From 4745f264b26705f8ce7f98072b47f37b5e18ea1a Mon Sep 17 00:00:00 2001 From: fiser_jun Date: Tue, 4 Aug 2026 14:34:00 +0800 Subject: [PATCH] --- .gitignore | 40 + README.md | 187 +++ .../reports/8bit-g64-baseline-2026-07-25.md | 61 + mlx_streaming/.gitignore | 3 + mlx_streaming/__init__.py | 0 mlx_streaming/cli.py | 329 +++++ mlx_streaming/config.py | 298 +++++ mlx_streaming/core/__init__.py | 0 mlx_streaming/core/cache/__init__.py | 1 + mlx_streaming/core/cache/autopin.py | 174 +++ mlx_streaming/core/cache/blob_loader.py | 238 ++++ mlx_streaming/core/cache/expert_store.py | 604 +++++++++ mlx_streaming/core/cache/kv_quant_patch.py | 116 ++ mlx_streaming/core/cache/quant_kv.py | 223 ++++ mlx_streaming/core/cache/resident_pool.py | 635 +++++++++ mlx_streaming/core/cache/virtual_pool.py | 237 ++++ mlx_streaming/core/linear_attn/__init__.py | 1 + .../linear_attn/gated_delta_multistate.py | 226 ++++ mlx_streaming/core/mem.py | 75 ++ mlx_streaming/core/moe/__init__.py | 1 + mlx_streaming/core/moe/block.py | 431 +++++++ mlx_streaming/core/moe/compute.py | 208 +++ mlx_streaming/core/moe/custom_kernel.py | 266 ++++ mlx_streaming/core/moe/gate.py | 32 + mlx_streaming/core/moe/native_moe.py | 504 ++++++++ mlx_streaming/core/prefetch/__init__.py | 1 + mlx_streaming/core/prefetch/bg_prefetch.py | 133 ++ mlx_streaming/core/prefetch/cross_layer.py | 181 +++ mlx_streaming/core/prefetch/native_staging.py | 229 ++++ mlx_streaming/core/prefetch/patch.py | 56 + mlx_streaming/core/profiling.py | 131 ++ mlx_streaming/core/route_trace.py | 50 + mlx_streaming/model_builder.py | 252 ++++ mlx_streaming/models/__init__.py | 0 mlx_streaming/mtp/__init__.py | 0 mlx_streaming/mtp/bench_verdict.py | 28 + mlx_streaming/mtp/drafter.py | 188 +++ mlx_streaming/mtp/generate.py | 506 ++++++++ mlx_streaming/mtp/kv_cache.py | 276 ++++ mlx_streaming/mtp/qwen3_next_mtp.py | 95 ++ .../native_moe_ext.cpython-314-darwin.so | Bin 0 -> 1805288 bytes mlx_streaming/prep/__init__.py | 0 mlx_streaming/prep/blob_layout.py | 32 + mlx_streaming/prep/download_retry.py | 35 + mlx_streaming/prep/extract_mtp.py | 131 ++ mlx_streaming/prep/pack_blob_from_experts.py | 87 ++ mlx_streaming/prep/pack_compute_buffers.py | 83 ++ mlx_streaming/prep/pack_expert_bundles.py | 70 + mlx_streaming/prep/pack_expert_ranges.py | 135 ++ mlx_streaming/prep/repack_expert_blobs.py | 92 ++ mlx_streaming/prep/split_experts.py | 69 + mlx_streaming/runtime/__init__.py | 1 + mlx_streaming/runtime/run_baseline.py | 42 + mlx_streaming/runtime/run_mmap.py | 69 + mlx_streaming/runtime/run_mtp_spec.py | 409 ++++++ mlx_streaming/runtime/run_spec.py | 116 ++ mlx_streaming/runtime/run_streaming.py | 180 +++ mlx_streaming/server/__init__.py | 9 + mlx_streaming/server/__main__.py | 95 ++ mlx_streaming/server/admin.py | 116 ++ mlx_streaming/server/app.py | 104 ++ mlx_streaming/server/openai.py | 338 +++++ mlx_streaming/server/state.py | 101 ++ mlx_streaming/tui/__init__.py | 9 + mlx_streaming/tui/app.py | 334 +++++ mlx_streaming/tui/backend.py | 436 +++++++ mlx_streaming/tui/banner.py | 3 + mlx_streaming/tui/styles.tcss | 41 + models/README.md | 49 + native/ext/CMakeLists.txt | 32 + native/ext/Makefile | 14 + native/ext/bindings.cpp | 153 +++ native/ext/common.h | 34 + native/ext/compute/fused_moe.cpp | 664 ++++++++++ native/ext/compute/fused_moe.h | 23 + native/ext/io/bg_reader.cpp | 183 +++ native/ext/io/bg_reader.h | 23 + native/ext/io/blob_io.h | 16 + native/ext/io/blob_load.cpp | 58 + native/ext/io/blob_load.h | 7 + native/ext/pool/demand.cpp | 331 +++++ native/ext/pool/demand.h | 29 + native/ext/pool/owned_pool.cpp | 89 ++ native/ext/pool/owned_pool.h | 14 + native/ext/pool/side_region.cpp | 389 ++++++ native/ext/pool/side_region.h | 21 + native/ext/prefetch/prefetch.cpp | 217 ++++ native/ext/prefetch/prefetch.h | 23 + pyproject.toml | 44 + uv.lock | 1129 +++++++++++++++++ 90 files changed, 13695 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 benchmarks/reports/8bit-g64-baseline-2026-07-25.md create mode 100644 mlx_streaming/.gitignore create mode 100644 mlx_streaming/__init__.py create mode 100644 mlx_streaming/cli.py create mode 100644 mlx_streaming/config.py create mode 100644 mlx_streaming/core/__init__.py create mode 100644 mlx_streaming/core/cache/__init__.py create mode 100644 mlx_streaming/core/cache/autopin.py create mode 100644 mlx_streaming/core/cache/blob_loader.py create mode 100644 mlx_streaming/core/cache/expert_store.py create mode 100644 mlx_streaming/core/cache/kv_quant_patch.py create mode 100644 mlx_streaming/core/cache/quant_kv.py create mode 100644 mlx_streaming/core/cache/resident_pool.py create mode 100644 mlx_streaming/core/cache/virtual_pool.py create mode 100644 mlx_streaming/core/linear_attn/__init__.py create mode 100644 mlx_streaming/core/linear_attn/gated_delta_multistate.py create mode 100644 mlx_streaming/core/mem.py create mode 100644 mlx_streaming/core/moe/__init__.py create mode 100644 mlx_streaming/core/moe/block.py create mode 100644 mlx_streaming/core/moe/compute.py create mode 100644 mlx_streaming/core/moe/custom_kernel.py create mode 100644 mlx_streaming/core/moe/gate.py create mode 100644 mlx_streaming/core/moe/native_moe.py create mode 100644 mlx_streaming/core/prefetch/__init__.py create mode 100644 mlx_streaming/core/prefetch/bg_prefetch.py create mode 100644 mlx_streaming/core/prefetch/cross_layer.py create mode 100644 mlx_streaming/core/prefetch/native_staging.py create mode 100644 mlx_streaming/core/prefetch/patch.py create mode 100644 mlx_streaming/core/profiling.py create mode 100644 mlx_streaming/core/route_trace.py create mode 100644 mlx_streaming/model_builder.py create mode 100644 mlx_streaming/models/__init__.py create mode 100644 mlx_streaming/mtp/__init__.py create mode 100644 mlx_streaming/mtp/bench_verdict.py create mode 100644 mlx_streaming/mtp/drafter.py create mode 100644 mlx_streaming/mtp/generate.py create mode 100644 mlx_streaming/mtp/kv_cache.py create mode 100644 mlx_streaming/mtp/qwen3_next_mtp.py create mode 100755 mlx_streaming/native_moe_ext.cpython-314-darwin.so create mode 100644 mlx_streaming/prep/__init__.py create mode 100644 mlx_streaming/prep/blob_layout.py create mode 100644 mlx_streaming/prep/download_retry.py create mode 100644 mlx_streaming/prep/extract_mtp.py create mode 100644 mlx_streaming/prep/pack_blob_from_experts.py create mode 100644 mlx_streaming/prep/pack_compute_buffers.py create mode 100644 mlx_streaming/prep/pack_expert_bundles.py create mode 100644 mlx_streaming/prep/pack_expert_ranges.py create mode 100644 mlx_streaming/prep/repack_expert_blobs.py create mode 100644 mlx_streaming/prep/split_experts.py create mode 100644 mlx_streaming/runtime/__init__.py create mode 100644 mlx_streaming/runtime/run_baseline.py create mode 100644 mlx_streaming/runtime/run_mmap.py create mode 100644 mlx_streaming/runtime/run_mtp_spec.py create mode 100644 mlx_streaming/runtime/run_spec.py create mode 100644 mlx_streaming/runtime/run_streaming.py create mode 100644 mlx_streaming/server/__init__.py create mode 100644 mlx_streaming/server/__main__.py create mode 100644 mlx_streaming/server/admin.py create mode 100644 mlx_streaming/server/app.py create mode 100644 mlx_streaming/server/openai.py create mode 100644 mlx_streaming/server/state.py create mode 100644 mlx_streaming/tui/__init__.py create mode 100644 mlx_streaming/tui/app.py create mode 100644 mlx_streaming/tui/backend.py create mode 100644 mlx_streaming/tui/banner.py create mode 100644 mlx_streaming/tui/styles.tcss create mode 100644 models/README.md create mode 100644 native/ext/CMakeLists.txt create mode 100644 native/ext/Makefile create mode 100644 native/ext/bindings.cpp create mode 100644 native/ext/common.h create mode 100644 native/ext/compute/fused_moe.cpp create mode 100644 native/ext/compute/fused_moe.h create mode 100644 native/ext/io/bg_reader.cpp create mode 100644 native/ext/io/bg_reader.h create mode 100644 native/ext/io/blob_io.h create mode 100644 native/ext/io/blob_load.cpp create mode 100644 native/ext/io/blob_load.h create mode 100644 native/ext/pool/demand.cpp create mode 100644 native/ext/pool/demand.h create mode 100644 native/ext/pool/owned_pool.cpp create mode 100644 native/ext/pool/owned_pool.h create mode 100644 native/ext/pool/side_region.cpp create mode 100644 native/ext/pool/side_region.h create mode 100644 native/ext/prefetch/prefetch.cpp create mode 100644 native/ext/prefetch/prefetch.h create mode 100644 pyproject.toml create mode 100644 uv.lock diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..958e6c6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,40 @@ +.venv/ +__pycache__/ +*.pyc +.pytest_cache/ +.worktrees/ +*.gputrace/ + +# 模型权重本地放置（仅提交 models/README.md） +models/** +!models/README.md +*.safetensors +/experts/ + +# CMake 缓存 +native/ext/build/ + +# 日志与一次性产物 +logs/ +*.log +*.bak +benchmarks/reports/*.json +/dl_*.sh +/dl_*.log +/build_*.sh +/build_*.log +/extract_mtp.log +/dl_model.py + +# 根目录校验脚本（非运行时） +/verify_*.py +# 本地可留、不入库 +mlx_streaming/tests/ +mlx_streaming/tools/ +benchmarks/*.py +.DS_Store + +# 本地可留、不入库 +/性能优化与选型*.md +/8bit*.md +/演示*.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..b3273c1 --- /dev/null +++ b/README.md @@ -0,0 +1,187 @@ +# Sparkle + +本地 **Qwen3-Next-80B-A3B Instruct 8bit** 流式推理引擎，对接 ccSparkle Agent（模型源：**Sparkle**）。 + +在约 64GB 统一内存的 Apple Silicon 上，把约 77GB 专家权重放在 SSD，运行时按需装入 + 预取，从而跑通 80B 8bit（oMLX / Ollama 全量加载做不到的档位）。动机见 [`models/README.md`](./models/README.md)：用 80B@64G 验证流式路径，对应下游 **30B 8bit A3B** 进 32G/16G、统一内存省 50%+。 + +| 项 | 值 | +|----|----| +| Python | **3.14.***（与 `native_moe_ext.cpython-314-darwin.so` 对齐；见 `pyproject.toml`） | +| 已验证机器 | MacBook Pro M2 Max / 64GB | +| 安装目录 | 本仓库根目录（下文记为 `$SPARKLE_HOME`，即 `clone` 后的路径） | +| 引擎端口 | `8317`（经中台 `8088` 启停与代理） | +| 模型 / 专家 / MTP | 见 [`models/README.md`](./models/README.md)（不入库，约 160GB） | +| 性能基线 | 稳态约 9–10 tok/s（贪心+MTP）；采样约 4–6 tok/s；常驻约 23–33GB 量级 | + +依赖安装走清华 PyPI 镜像（`pyproject.toml` 的 `tool.uv.index`）。 + +--- + +## 1. 系统组成 + +``` +浏览器前端(8090) ← 日常对话界面 + │ 顶部选 Sparkle 源 + ▼ +中台代理(8088) ← 转发请求，规避跨域 + ▼ +Sparkle 引擎(8317) ← 80B 8bit 流式推理 + ├─ /v1/chat/completions OpenAI 兼容接口 + └─ /admin 调参页（槽位 / 投机宽度 / 状态） +``` + +专家权重留在 SSD，运行时按需读取 + 预测式预取；首次启动和切换话题时偶有停顿（读盘）。 + +**AUTOPIN**（默认开启）：记录专家路由热度（`models/experts_8bit_g64/pool_usage.json`），启动时把最热一批 pin 进常驻池（默认占槽位 50%，定死不驱逐）。冷启动命中率可从约 72% 升到 **98%+**。`AUTOPIN=0` 关闭；`AUTOPIN_BUDGET_FRAC` 调比例（0–1）。 + +--- + +## 2. 启动 + +**推荐**：打开对话界面 → 顶部选 **Sparkle** → 中台自动加载引擎（约 90 秒，状态变绿后再对话）。 + +1. 中台（桌面应用常会自动拉起；路径按你本机 Agent 工程根目录，下文记 `$AGENT_HOME`）： + ```bash + cd "$AGENT_HOME" && node scripts/zhongtai-server.js + ``` +2. 前端静态服务： + ```bash + cd "$AGENT_HOME/<前端静态目录>" && python3 -m http.server 8090 + ``` +3. 浏览器打开对话页，顶部点 **Sparkle**——未运行时会自动启动（也可在 `配` → Sparkle tab「引擎管理」点**加载**）。 + +就绪：`配` → Sparkle 状态点变绿，或 `curl -s http://127.0.0.1:8088/sparkle-engine/status` 见 `"loaded":true`。 + +**手动启动（调试）**： + +```bash +cd "$SPARKLE_HOME" # 本仓库根目录 +.venv/bin/python -m mlx_streaming.server --port 8317 \ + --model models/Qwen3-Next-80B-A3B-Instruct-MLX-8bit \ + --expert-dir models/experts_8bit_g64 \ + --qn-config models/Qwen3-Next-80B-A3B-Instruct-MLX-8bit/config.json \ + --mtp-out models/qn_mtp_weights.safetensors \ + --expert-slots 96 +``` + +- Admin：`http://127.0.0.1:8317/admin` +- 健康：`curl -s http://127.0.0.1:8317/api/stats`（`unplaced_experts` 应为 `0`） + +中台默认 `SPARKLE_ENGINE_DIR` → 本目录。改目录名后若「卸载」失灵，先杀 `8317` 再重启中台。 + +> **工具白名单**：默认只放画像 4 工具（`profile_list_employees` / `profile_get_portrait_overview` / `profile_get_person_report` / `profile_get_team_report`）。中台动态工具集会改 system 头部哈希，导致前缀快照 miss；钉死工具集后快照才稳定。排班 `metro_*` 等会被裁掉。临时放开：`TOOLS_ALLOW=`；换名单：`TOOLS_ALLOW=a,b,c`。裁剪结果见 `logs/server.log` 的 `[TOOLS_ALLOW]`。 + +> **不要自己设 `PREFILL_CHUNK`**。安全上限由槽位决定；引擎启动时按槽位自算（96 槽 → 4，64 槽 → 3），超限会被钳制并打日志。 + +--- + +## 3. 日常对话 + +1. 打开对话页（`http://localhost:8090`），顶部选 **Sparkle** +2. 首次：`配` → Sparkle tab：Base URL `http://127.0.0.1:8317`，API Key 任意非空（如 `sparkle`），刷新选模型 `Qwen3-Next-80B-A3B-Instruct-MLX-8bit`，保存 +3. 正常对话；工具调用链路与 oMLX 源相同 + +--- + +## 4. 调参与引擎管理 + +`配` → **本地 Sparkle** tab「引擎管理」（与 `http://127.0.0.1:8317/admin` 等价）： + +状态点：灰（未启动）→ 黄（加载中，约 90 秒）→ 绿（就绪）。`加载` / `卸载` 启停；就绪时显示 tok/s 与 RSS。 + +| 参数 | 范围 | 说明 | 生效 | +|------|------|------|------| +| expert-slots | 32–160 | 常驻专家数，越大命中率越高、越占内存（每槽 ~160MB）。64GB + AUTOPIN：64 槽约 9 tok/s；**96 槽约 10.3 tok/s（推荐）**；≥128 近红线；**160 会卡死，勿用** | 应用后进程级重启（约 90 秒） | +| K（投机宽度） | 1–5 | MTP 草稿 token 数，默认 3；只影响速度 | 即时 | +| max_tokens | — | 单次回答最大长度 | 即时 | + +日常 64 够用；长文档可试 96–128；系统开始 swap 就降回 64。 + +--- + +## 5. 故障排查 + +| 现象 | 原因 | 处理 | +|------|------|------| +| 503 / engine reloading | 加载或重建中 | 等 60–90 秒后再试 | +| `EADDRINUSE ... 8317` | 旧 server 未关 | `lsof -tiTCP:8317 -sTCP:LISTEN \| xargs kill` | +| 中台 `EADDRINUSE ... 8088` | 已有健康中台 | 直接用；要重启先 kill | +| 改名后「卸载」无效 | 旧进程挂在旧路径 | 杀 8317，重启中台再加载 | +| 明显变慢 | swap 或槽位太低 | 看内存压力；调槽；关其他大应用 | +| 首轮特别慢 | 冷启动 | 正常，第二轮起恢复 | +| 长文中途停顿 | 读盘装冷门专家 | 正常；调高槽位可缓解 | +| 刷新拉不到模型 | server 未起或端口错 | `curl http://127.0.0.1:8317/v1/models -H "Authorization: Bearer x"` | + +--- + +## 6. 常见问题 + +**Q: 为什么 oMLX / Ollama 里看不到这个 8bit 模型？** +A: 权重远超 64GB 整模上限，只有 Sparkle 流式能跑。oMLX 继续跑中小模型即可。 + +**Q: 投机宽度 K 是什么？** +A: MTP 每步草稿 token 数；主模型批量验证，猜中白赚。只影响速度，不影响内容。 + +**Q: 回答乱编 / 不调工具 / 幻觉？** +A: Qwen3-Next 官方要求采样（temp 0.7 / top_p 0.8 / top_k 20）。贪心+MTP 在该模型上易幻觉。默认走采样（约 4–6 tok/s）；显式 `temperature: 0` 才走贪心+MTP（约 10 tok/s）。Sparkle 客户端对真实对话会把 `0` 改回采样。 + +**Q: 模块分析比 oMLX 7B 还空？** +A: 多为分析轮传了 `temperature: 0` 进贪心导致指令遵循崩坏，不是花名册数据错。另查 `unplaced_experts`（非 0 = 错专家）与是否需清空 `prefix_snapshots`。 + +**Q: 回答里夹「�」乱码？** +A: 已修：byte-level BPE 流式反分词用 `StreamingDetokenizer`，避免半字符 U+FFFD 丢字。 + +**Q: 8bit 比 7B 还笨？** +A: 曾因 prefill 超槽静默映射到 0 号专家。现自动算 `PREFILL_CHUNK`、超容量走正确慢路径，并用 `unplaced_experts` 告警（健康值恒为 0）。非 0 时回答不可信；若错算过，删 `models/prefix_snapshots/` 再预热。 + +**Q: 和云端略有不同？** +A: 量化 + KV 压缩的正常差异；语义质量不受影响。 + +**Q: 首轮为什么要等？** +A: 新对话要 prefill 系统提示+工具定义。有前缀快照后：重启后首会话约百秒级，之后新会话约数十秒，同会话追问约数秒。`PREFIX_SNAPSHOT_HEAD` 调长度（默认 2048，0 关闭）。 + +**Q: 速度还能再快吗？** +A: 96 槽 + AUTOPIN + prefill 分块约 9–10 tok/s，接近本机 8bit 上限。160 槽会卡死，勿试。 + +**Q: 能跑多长上下文？** +A: KV 量化默认开；对话侧另有约 24k token 压缩阈值（中台配置）。 + +**Q: 想换模型或其它量化档？** +A: 本引擎面向 Qwen3-Next-80B 流式 8bit 部署；换档需按当前模型重新准备专家 blob 与 MTP。 + +--- + +## 7. 目录要点 + +``` +sparkle/ +├── mlx_streaming/ # 引擎 Python 包（含已编译 native_moe_ext*.so） +├── native/ext/ # C++/Metal 扩展源码（重编 .so 用） +├── models/ # 权重与专家 blob（约 160GB，勿删） +│ ├── Qwen3-Next-80B-A3B-Instruct-MLX-8bit/ +│ ├── experts_8bit_g64/ +│ └── qn_mtp_weights.safetensors +├── .venv/ +├── pyproject.toml / uv.lock +└── benchmarks/reports/ # 性能基线报告 +``` + +`models/` 三者缺一不可。不要把模型目录移进 oMLX 等其它运行时的模型扫描目录（会列出但整模加载不了）。 +换 Python 小版本时需 `make -C native/ext native_moe_ext` 重编 `.so`。`native/ext/build/` 已忽略。 + +--- + +## 8. 质量与速度（摘要） + +- **默认应对话走采样解码**（约 temperature 0.7 / top_p 0.8 / top_k 20）。`temperature=0` 会进贪心+MTP，本机实测指令遵循差于同表 oMLX 7B。 +- 稳态 decode：采样约 **4–6 tok/s**；贪心+MTP 约 **9–10 tok/s**（速度更高，但质量风险大，生产默认勿用）。 +- 专家池命中率健康时约 **94%+**；`unplaced_experts ≠ 0` 表示曾用错专家权重，回答不可信。 + +--- + +## 9. 相关文档 + +| 文档 | 内容 | +|------|------| +| [models/README.md](./models/README.md) | 为何做流式、NAS 下载与专家/MTP 准备 | +| [benchmarks/reports/8bit-g64-baseline-2026-07-25.md](./benchmarks/reports/8bit-g64-baseline-2026-07-25.md) | 8bit 性能基线 | diff --git a/benchmarks/reports/8bit-g64-baseline-2026-07-25.md b/benchmarks/reports/8bit-g64-baseline-2026-07-25.md new file mode 100644 index 0000000..7b8d775 --- /dev/null +++ b/benchmarks/reports/8bit-g64-baseline-2026-07-25.md @@ -0,0 +1,61 @@ +# Qwen3-Next-80B-A3B 8bit(g64)基线 — M2 Max 64GB + +日期:2026-07-25 · 机器:MacBook Pro M2 Max / 64GB / 内置 SSD +模型:`models/Qwen3-Next-80B-A3B-Instruct-MLX-8bit`(**bits=8, group_size=64**,79GB) +数据:`models/experts_8bit_g64/blobs`(48 层,77GB,v1 affine,stride 3,342,336B,page-aligned)+ `models/qn_mtp_weights.safetensors`(3.3GB) + +## 性能 + +| 场景 | tok/s | accept_len | 备注 | +|---|---|---|---| +| 冷启动首轮(32 tok) | 5.0 | 1.889 | 含 kernel 编译 + 池冷填充 | +| 稳态(128 tok,64 槽) | **9.4** | 2.246 | 符合 8-12 tok/s 预期 | +| server 短请求(33 tok,含 prefill) | 6.6 | — | OpenAI 链路实测 | + +- 常驻内存:RSS ≈ **23.5 GB**(`--expert-slots 64`,池 ~10GB + dense + 缓冲;64GB 机器余量充足,可上探 96-128 槽) +- 加载时间:约 60-90s(含预热) + +## 正确性(三层证据链) + +1. **kernel 单元等价**(`verify_8bit_equiv.py`):L0/L15/L47 blob fused kernel vs Python 参考 **diff = 0** +2. **端到端对拍**(`verify_e2e_spec.py`):贪心 vs 投机 token 17 确定性分叉 → 触发对照 +3. **KV 量化对照**(`verify_teacher_forced.py`,seq=1 vs seq=2 生产原语): + - `KV_QUANT=0`:**24/24 token 全一致**,cosine 0.99935 → 8bit 权重路径数值干净 + - `KV_QUANT=1`(生产默认):23/24(95.8%),cosine 0.99870,翻转处 margin=0 近 tie → 达质量门(≥95%/≥0.99) + - 结论:分叉全部归因于 KV 量化(KV 量化既有有损特性),与权重量化位宽无关 + +## 已知事项 + +- `VirtualPool.acquire_host` latent bug:单次前向单层唯一专家数超池容量时 fetch 回退会 AttributeError;生产 prefill 分块 chunk=2 永不触发。勿用整段 seq=N 一次性前向做诊断。 +- server `/admin` 调参页池内存估算已按 g64 修正为 ~160MB/槽。 +- server 模型 id 取目录名:`Qwen3-Next-80B-A3B-Instruct-MLX-8bit`。 + +## 复现命令 + +```bash +# CLI +.venv/bin/sparkle --model models/Qwen3-Next-80B-A3B-Instruct-MLX-8bit \ + --expert-dir models/experts_8bit_g64 \ + --qn-config models/Qwen3-Next-80B-A3B-Instruct-MLX-8bit/config.json \ + --mtp-out models/qn_mtp_weights.safetensors --expert-slots 64 --stats + +# OpenAI server(前端 Sparkle 源对接,端口 8317) +.venv/bin/python -m mlx_streaming.server --port 8317 \ + --model models/Qwen3-Next-80B-A3B-Instruct-MLX-8bit \ + --expert-dir models/experts_8bit_g64 \ + --qn-config models/Qwen3-Next-80B-A3B-Instruct-MLX-8bit/config.json \ + --mtp-out models/qn_mtp_weights.safetensors --expert-slots 64 +``` + +## 追记(2026-07-25 晚):AUTOPIN + 调优后 + +| 配置 | decode tok/s | 备注 | +|---|---|---| +| 64 槽 + AUTOPIN | ~8.0-9.4 | 稳定,RSS ~24GB | +| **96 槽 + AUTOPIN(生产)** | **9.3-10.3** | 甜点位,RSS ~32GB;CLI 128tok: 9.3(MTP4)/9.7(MTP8) | +| 160 槽 + AUTOPIN | 不可用 | RSS ~34GB 系统耗尽卡死(AUTOPIN eager 预热多占 ~5GB) | + +- AUTOPIN(热度持久化+启动 pin 定死):命中率 72.2%→98.6%,pin 3439 专家/6.5s +- PREFILL_CHUNK=8:676 tok prompt 2m49s→33s +- MTP_BITS 4→8 A/B:accept_len 2.246→2.207,无效,保持 4 +- 槽位调整改进程级重启(进程内重建在新 C++ pin 状态下会把引擎拖到 0.24 tok/s) diff --git a/mlx_streaming/.gitignore b/mlx_streaming/.gitignore new file mode 100644 index 0000000..77ac754 --- /dev/null +++ b/mlx_streaming/.gitignore @@ -0,0 +1,3 @@ +.venv/ +__pycache__/ +*.pyc diff --git a/mlx_streaming/__init__.py b/mlx_streaming/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mlx_streaming/cli.py b/mlx_streaming/cli.py new file mode 100644 index 0000000..179ac97 --- /dev/null +++ b/mlx_streaming/cli.py @@ -0,0 +1,329 @@ +"""sparkle 命令行:面向用户的交互式多轮对话(MTP 自投机快路径)。 + +用法示例: + sparkle # 直接进入交互式对话(默认子命令 chat) + sparkle chat # 同上 + sparkle -k 4 -n 800 --stats # 调宽投机、加长生成、每轮打印吞吐 + sparkle --system "你是一个简洁的助手" + sparkle --model models/Qwen3-Next-80B-A3B-Instruct-MLX-8bit --expert-slots 32 + +只做「生成」一件事:走 MTP 自投机 + 零拷贝双源侧区快路径。关键参数做成命令行 flag, +其余调优项仍从环境变量读取(见 mlx_streaming/config.py)。 + +交互期间可用命令: + /exit 或 /quit 退出 + /reset 清空对话历史(保留 system) + /help 打印帮助 +""" +import argparse +import sys +import time + +from mlx_streaming import config + +# MTP 快路径环境变量兜底配方(benchmark 验证过的最优组合)。 +# 用 setdefault 兜底:用户显式导出的环境变量优先级更高,不会被覆盖。 +# MTP_ADAPTIVE_DEPTH:置信度门控动态深度。逐位累计置信度跌破 tau 即停,低置信步抽浅省专家加载 +# (本系统 IO 瓶颈)。消融(reports/adaptive-depth-2026-07-05)证 τ=0.3、depth_max=3 纯向下收缩 +# +5~6% tok/s 且 bit-lossless、零额外显存,稳定优于最小树 pos0 救回 → 设为用户主路径默认。 +# depth_max=3 与基础 K 一致,在生产 EXPERT_SLOTS=32 下 seq·top_k 不溢出 cap(扩到 4 须 slots>=40)。 +# 注:动态深度与 TREE_TOP2 互斥(adaptive 仅在非 tree 的 plain 路径生效),故此处不开 TREE_TOP2。 +# KV_QUANT:IsoQuant K4/V3 + SO(4) 块旋转,仅作用于 12 个全注意力层(线性层递归态不动)。 +# 极致压缩长上下文 KV:128k 3.0→~0.68 GiB;短会话收益小但无害。K4/V3/旋转均取默认值, +# 开 KV_QUANT=1 即整套生效。质量验收:token 一致率≥95% + logits cosine≥0.99。 +_FASTPATH_ENV = { + "STREAM_BLOB_LOADER": "1", + "NATIVE_FUSED_PREFETCH": "1", + "ZEROCOPY_DUAL_SOURCE": "1", + "SIDEREGION_LFU": "1", + "KV_QUANT": "1", + "MTP_ADAPTIVE_DEPTH": "1", + "MTP_CONF_TAU": "0.3", + "MTP_DEPTH_MAX": "3", +} + + +def _build_engine(args, on_status=None): + """按 MTP 快路径装配 model / tokenizer / drafter。 + + 注意:model_builder 在 import 时就读 MODEL/EXPERT_DIR 等环境变量,所以必须 + 先把命令行参数写进 os.environ,再 import build_streaming_model。 + + on_status:可选进度回调;为 None 时进度打到 stderr(保持旧行为)。 + """ + def _emit(msg): + if on_status is not None: + on_status(msg) + else: + print(msg, file=sys.stderr, flush=True) + + import os + + os.environ["MODEL"] = args.model + os.environ["QN_CONFIG"] = args.qn_config + os.environ["MTP_OUT"] = args.mtp_out + os.environ["EXPERT_DIR"] = args.expert_dir + os.environ["EXPERT_SLOTS"] = str(args.expert_slots) + spec = args.spec_slots if args.spec_slots is not None else args.expert_slots + os.environ["POOL_SPEC_SLOTS"] = str(spec) + for k, v in _FASTPATH_ENV.items(): + os.environ.setdefault(k, v) + + import json + + import mlx.core as mx # noqa: F401 确保 MLX 已就绪 + from mlx_lm.models.qwen3_next import ModelArgs + + from mlx_streaming.core.mem import setup_memory_hygiene + from mlx_streaming.model_builder import build_streaming_model + from mlx_streaming.mtp.drafter import MTPDrafter + from mlx_streaming.mtp.qwen3_next_mtp import load_mtp + + # 长会话内存防御:封顶 MLX 可回收缓冲(默认 1GB),防长对话里缓冲缓存膨胀把常驻推过墙 / + # 触发 macOS 压缩器抖动。TUI 是典型长会话场景,故在用户主路径启动时就设上。 + _applied = setup_memory_hygiene(cache_gb=config.mlx_cache_limit_gb(), + wired_gb=config.mlx_wired_limit_gb()) + if _applied: + _emit(f"内存防御: {_applied}") + + _emit("正在加载主模型 + 专家(流式)...") + model, tok, _store = build_streaming_model() + _emit("正在加载 MTP drafter...") + with open(args.qn_config) as f: + margs = ModelArgs.from_dict(json.load(f)) + mtp = load_mtp(margs, args.mtp_out, quantize=True, bits=config.mtp_bits()) + mtp.embed_tokens = model.model.embed_tokens # 共享主模型 embedding + drafter = MTPDrafter(mtp, model.lm_head) + return model, tok, drafter + + +def _warmup(model, tok, drafter, args): + """跑一次生成做预热:首轮的明显卡顿主要来自现编译 Metal kernel + 填 MoE 专家 resident 池, + 提前把这部分一次性开销移到加载阶段。 + + 覆盖增强:用一段**较长、token id 跨大跨度词表分散**的合成 prompt——MoE 路由依赖 token 内容, + 分散的 id 会命中更多专家、更充分地预填专家池;较长 prompt 又能走通多块分块 prefill。 + 合成 id 直接走 ids_mode(不依赖 tokenizer),分块 prefill(chunk=2)保证长 prompt 也不抬高显存峰值。 + 预热失败不致命(直接吞掉异常),不影响后续真实生成。 + """ + import mlx.core as mx + + from mlx_streaming.mtp.generate import mtp_generate + + try: + vocab = int(model.model.embed_tokens.weight.shape[0]) + n = min(64, vocab) # 预热 prompt 长度(兼顾覆盖与耗时) + step = max(1, vocab // n) # 在词表内均匀取样,最大化专家覆盖 + ids = [(1 + i * step) % vocab for i in range(n)] + mtp_generate(model, drafter, tok, mx.array([ids]), 8, + K=args.k, ids_mode=True) + except Exception: # noqa: BLE001 预热仅为压首轮延迟,失败不应中断启动 + pass + + +def _encode_chat(tok, messages, tools=None): + """把多轮对话按聊天模板编码成 token id 列表。tools(OpenAI 格式)交给模板渲染工具定义。""" + tmpl = getattr(tok, "chat_template", None) + if tmpl: + kwargs = {"add_generation_prompt": True} + if tools: + kwargs["tools"] = tools + out = tok.apply_chat_template(messages, **kwargs) + # 返回形态随 transformers 版本而变,三种都要兜住: + # transformers 4.x → list[int] + # transformers 5.x(默认) → BatchEncoding,即 {"input_ids": [[...]], ...} + # 部分实现 → str + # 只写 list(out) 会在 5.x 下取到字典的**键名** ["input_ids", "attention_mask"], + # 于是 prompt 退化成两个字符串:主生成路径(backend.py 的 ids)喂进去全是乱码, + # 且 _fixed_prefix_len 恒为 2 → head < 256 → 前缀快照永久失效。 + # 单元测试全部 monkeypatch 掉了本函数,捕不到这个回归,故在此显式判型。 + if isinstance(out, str): + return tok.encode(out) + ids = out if isinstance(out, (list, tuple)) else out["input_ids"] + if len(ids) > 0 and isinstance(ids[0], (list, tuple)): + ids = ids[0] # 批次维度 + return [int(t) for t in ids] + # 无聊天模板:朴素拼接兜底。 + text = "" + for m in messages: + text += f"{m['role']}: {m['content']}\n" + text += "assistant: " + return tok.encode(text) + + +def _eos_set(tok): + """收集所有可能的结束符 token id。""" + eos = set() + ids = getattr(tok, "eos_token_ids", None) + if ids: + eos |= set(ids) + one = getattr(tok, "eos_token_id", None) + if one is not None: + eos.add(one) + return eos + + +def _truncate_eos(produced, eos): + """遇到第一个结束符即截断(不含结束符本身)。""" + for i, t in enumerate(produced): + if t in eos: + return produced[:i] + return produced + + +_HELP = """可用命令: + /exit, /quit 退出 + /reset 清空对话历史(保留 system) + /help 显示本帮助 +直接输入文本即可对话。""" + + +def cmd_chat(args): + """默认启动全屏 TUI;--plain 走纯文本 REPL;--demo 用假后端免模型预览界面。""" + if getattr(args, "plain", False): + return _chat_repl(args) + from mlx_streaming.tui import run_tui + if getattr(args, "demo", False): + # 免模型预览:秒开 TUI,假流式回答,用于验证界面/占位符/状态栏 + from mlx_streaming.tui.backend import FakeBackend + demo = FakeBackend( + reply="这是 --demo 测试回答:界面、逐字流式、状态栏(token 数 / tok·s)" + "均为模拟,不加载模型。按 Esc 可中断,/help 看命令。", + delay=0.03) + return run_tui(demo, args) + from mlx_streaming.tui.backend import MLXBackend + return run_tui(MLXBackend(args), args) + + +def _chat_repl(args): + model, tok, drafter = _build_engine(args) + + import mlx.core as mx + + from mlx_streaming.mtp.generate import mtp_generate + from mlx_streaming.tui.backend import _reuse_prefix_len + + eos = _eos_set(tok) + base_messages = [] + if args.system: + base_messages.append({"role": "system", "content": args.system}) + messages = list(base_messages) + # 跨轮复用:持久化上一轮 main_cache 及其对应的 token 序列(与 MLXBackend 同机制)。 + main_cache = None + cached_ids: list[int] = [] + + print("\n正在预热(编译 kernel + 填专家池)…", file=sys.stderr, flush=True) + _warmup(model, tok, drafter, args) + print("模型已就绪。输入 /help 查看命令,/exit 退出。", file=sys.stderr, flush=True) + + while True: + try: + user = input("\n你 > ").strip() + except (EOFError, KeyboardInterrupt): + print() + break + if not user: + continue + if user in ("/exit", "/quit"): + break + if user == "/reset": + messages = list(base_messages) + main_cache, cached_ids = None, [] # 清历史同时弃用旧 cache + print("对话历史已清空。", file=sys.stderr) + continue + if user == "/help": + print(_HELP, file=sys.stderr) + continue + + messages.append({"role": "user", "content": user}) + ids = _encode_chat(tok, messages) + + # 旧 cache 是本轮 prompt 的严格前缀时只 prefill 新增后缀,否则全量重建。 + cached_len = (_reuse_prefix_len(cached_ids, ids) + if main_cache is not None else 0) + cur_cache = main_cache if cached_len else model.make_cache() + + # EOS 提前停止:否则会空跑到 max_tokens,且 produced 会含 EOS 后垃圾 token, + # 使 cached_ids 与下轮编码前缀断裂、复用永不触发。 + produced_all: list[int] = [] + + def _on_tokens(new_ids): + produced_all.extend(new_ids) + truncated = _truncate_eos(produced_all, eos) + return len(truncated) < len(produced_all) + + t0 = time.perf_counter() + produced, stats = mtp_generate( + model, drafter, tok, mx.array([ids]), + args.max_tokens, K=args.k, ids_mode=True, profile=args.stats, + on_tokens=_on_tokens, main_cache=cur_cache, cached_len=cached_len) + dt = time.perf_counter() - t0 + + # offset 对账:仅无 over-commit 时记录 cache 供下轮复用(见 MLXBackend.generate)。 + if stats.get("resident_tokens") == len(ids) + len(produced) - 1: + main_cache = cur_cache + cached_ids = list(ids) + list(produced[:-1]) + else: + main_cache, cached_ids = None, [] + + out_ids = _truncate_eos(produced, eos) + text = tok.decode(out_ids) + print(f"\n助手 > {text}") + messages.append({"role": "assistant", "content": text}) + + if args.stats: + tps = len(out_ids) / dt if dt > 0 else 0.0 + print(f"[{len(out_ids)} tok, {tps:.1f} tok/s, " + f"accept_len={stats.get('avg_accept_len')}]", file=sys.stderr) + + print("再见。", file=sys.stderr) + return 0 + + +def _add_chat_args(p): + p.add_argument("--model", default=config.model_path(), help="主模型路径(MLX 量化)") + p.add_argument("--expert-dir", default=config.expert_dir(), + help="拆分后的 per-expert 目录") + p.add_argument("--mtp-out", default=config.mtp_out(), help="MTP 权重文件") + p.add_argument("--qn-config", default=config.qn_config(), + help="Qwen3-Next 配置 JSON") + p.add_argument("-k", "--k", type=int, default=3, help="MTP 投机宽度(默认 3)") + p.add_argument("-n", "--max-tokens", type=int, default=4096, + help="每轮最多生成的新 token 数(默认 4096)") + p.add_argument("--expert-slots", type=int, default=32, + help="常驻专家池容量(默认 32,同时作为侧区行数默认)") + p.add_argument("--spec-slots", type=int, default=None, + help="侧区行数 POOL_SPEC_SLOTS(默认跟随 --expert-slots)") + p.add_argument("--system", default=None, help="可选 system 提示词") + p.add_argument("--stats", action="store_true", + help="每轮结束在 stderr 打印 token 数 / tok·s / 接受长度") + p.add_argument("--plain", action="store_true", + help="用纯文本 REPL,不启动全屏 TUI(终端不兼容/调试时用)") + p.add_argument("--demo", action="store_true", + help="免模型预览:用假后端秒开 TUI,验证界面/流式/状态栏") + p.set_defaults(func=cmd_chat) + + +def _build_parser(): + parser = argparse.ArgumentParser( + prog="sparkle", + description="sparkle:Apple Silicon 上的流式 MoE + Qwen3-Next MTP 自投机推理") + sub = parser.add_subparsers(dest="cmd") + chat = sub.add_parser("chat", help="进入交互式多轮对话(MTP 自投机快路径)") + _add_chat_args(chat) + return parser + + +def main(argv=None): + argv = list(sys.argv[1:] if argv is None else argv) + subcmds = {"chat"} + # 让 chat 成为默认子命令:不带子命令(或首参是 flag)时自动补上 chat; + # 但保留顶层 -h/--help 直接显示总帮助。 + if not argv or (argv[0] not in subcmds and argv[0] not in ("-h", "--help")): + argv = ["chat"] + argv + args = _build_parser().parse_args(argv) + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/mlx_streaming/config.py b/mlx_streaming/config.py new file mode 100644 index 0000000..4d46ae0 --- /dev/null +++ b/mlx_streaming/config.py @@ -0,0 +1,298 @@ +"""集中配置：所有运行时开关从环境变量读取（env 名与默认值与历史一致，便于现有跑法）。 + +设计目的：把原先散落在 core/ 各文件的 `os.environ.get` 全部收口到这里，单点可查、 +避免默认值漂移。读取均为运行时（每次调用读 env），与原行为一致——测试/probe 用 +monkeypatch env 仍生效，热路径每 token 读 env 的成本与原先相同。 + +对「同一 env 名在不同调用方有不同默认」的项（如 CROSS_LAYER_PREFETCH_AHEAD：hook=0、 +native 预取=1），accessor 暴露 `default` 参数，由调用方传入，绝不擅自统一。 +""" +import os + + +def _b(name: str, default: str = "0") -> bool: + return os.environ.get(name, default) == "1" + + +def _i(name: str, default) -> int: + return int(os.environ.get(name, str(default))) + + +def _f(name: str, default) -> float: + return float(os.environ.get(name, str(default))) + + +def _s(name: str, default: str = "") -> str: + return os.environ.get(name, default) + + +# ============================ 模型 / 目录 ============================ +# 默认放持久目录 models/（不要放 /tmp，会被系统清空）。运行从仓库根目录起，相对路径即可。 +def model_path() -> str: return _s("MODEL", "models/Qwen3-Next-80B-A3B-Instruct-MLX-8bit") +def qn_config() -> str: return _s("QN_CONFIG", "models/Qwen3-Next-80B-A3B-Instruct-MLX-8bit/config.json") +def mtp_out() -> str: return _s("MTP_OUT", "models/qn_mtp_weights.safetensors") +def expert_dir(default: str = "models/experts_8bit_g64") -> str: return _s("EXPERT_DIR", default) +def expert_slots() -> int: return _i("EXPERT_SLOTS", 64) +# 长期运行内存防御:封顶 MLX 可回收缓冲(默认 1GB),防长会话缓存膨胀; +# 双源侧区池的专家 buffer 走 C++ owned pool、不经 MLX 缓冲缓存,故 1GB 缓冲复用额度已够, +# 再大只是白占常驻。wired limit 默认 0=关(opt-in),设 >0 则 wire 该 GB 数的 GPU 缓冲防 macOS +# 压缩器,须 < 系统建议工作集(本机 26.8GB)。 +def mlx_cache_limit_gb() -> float: return _f("MLX_CACHE_LIMIT_GB", 1.0) +def mlx_wired_limit_gb() -> float: return _f("MLX_WIRED_LIMIT_GB", 0.0) +def expert_pool_profile() -> str: return _s("EXPERT_POOL_PROFILE", "") +def hidden_variant() -> str: return _s("HIDDEN_VARIANT", "pre_final_norm") +def blob_dir() -> str: return _s("BLOB_DIR", "") +def compute_buffer_dir() -> str: return _s("COMPUTE_BUFFER_DIR", "") + + +# ============================ MoE 热路径 ============================ +def resident_pool_enabled() -> bool: return _b("RESIDENT_POOL", "1") +def gpu_remap_enabled() -> bool: return _b("GPU_REMAP", "1") +# 实验：让 verify(seq>1)也走 GPU 侧 slot 重映射(acquire_gpu),消掉 host 路径每层 .tolist() 栅栏。 +def verify_gpu_remap() -> bool: return _b("VERIFY_GPU_REMAP", "0") +# A2：GPU 重映射路径下 promote 是否用 GPU membership 现算 used 过滤假阳性（默认开；回退设 0）。 +def gpu_remap_promote_filter() -> bool: return _b("GPU_REMAP_PROMOTE_FILTER", "1") +def moe_topk_override() -> "str | None": return os.environ.get("MOE_TOPK_OVERRIDE") +def eager_expert_load() -> bool: return _b("EAGER_EXPERT_LOAD", "0") + + +# ============================ 缓存 / 驱逐 ============================ +def evict_policy() -> str: return _s("EVICT_POLICY", "lfu") +# 专家读盘是否绕过 OS page cache(F_NOCACHE):默认开,保证基准每次都是真实 NVMe 读、 +# 结果可复现(不被页缓存冷热污染);EXPERT_NOCACHE=0 恢复 mmap/page-cache(重复跑更快但飘)。 +def expert_nocache() -> bool: return _b("EXPERT_NOCACHE", "1") +def lfu_decay_interval() -> int: return _i("LFU_DECAY_INTERVAL", 0) +def expert_bundle() -> bool: return _b("EXPERT_BUNDLE", "0") +def expert_bundle_dir(default: str) -> str: return _s("EXPERT_BUNDLE_DIR", default) +def expert_bundle_cache() -> int: return _i("EXPERT_BUNDLE_CACHE", 4) +def expert_stack_cache() -> int: return _i("EXPERT_STACK_CACHE", 0) +def async_prefetch() -> bool: return _b("ASYNC_PREFETCH", "0") +def prefetch_buffer_experts() -> int: return _i("PREFETCH_BUFFER_EXPERTS", 2048) + + +# ============================ AUTOPIN(专家热度持久化 + 启动预热钉死) ============================ +# 总开关。1(默认)：会话中持续累计每层路由热度并节流落盘({EXPERT_DIR}/pool_usage.json)， +# 引擎启动时按历史热度把每层最热专家预填进常驻池并 pin 住(不参与任何驱逐)，消灭冷启动慢热。 +# 0：整个特性零行为(不计数、不读写盘、不预热)。 +def autopin() -> bool: return _b("AUTOPIN", "1") +# 热度文件路径覆盖：缺省 {EXPERT_DIR}/pool_usage.json，结构 {layer: {expert_id: count}}。 +def autopin_usage_file() -> str: return _s("AUTOPIN_USAGE_FILE", "") +# 落盘节流：约每这么多次前向写一次盘(前向边界按「MoE 层号回绕」近似)；atexit 总会兜底写一次。 +# 写同目录临时文件再 rename 防半写损坏；读不到/损坏静默从零开始。 +def autopin_save_every() -> int: return _i("AUTOPIN_SAVE_EVERY", 200) +# 预热 pin 预算占比：每层 pin 数量 = floor(该层池容量 cap_for × FRAC)，取值 0-1，默认 0.5。 +# pinned 永占真实区槽：FRAC 过大会挤掉 demand 自由槽(驱逐候选变少)，不建议 >0.5。 +def autopin_budget_frac() -> float: return min(1.0, max(0.0, _f("AUTOPIN_BUDGET_FRAC", 0.5))) + + +# ============================ 自定义 Metal 算子 ============================ +def custom_qproj() -> bool: return _b("CUSTOM_QPROJ", "0") +def custom_qproj_bits() -> int: return _i("CUSTOM_QPROJ_BITS", 6) +def custom_qproj_targets() -> str: return _s("CUSTOM_QPROJ_TARGETS", "gate,up") +def custom_qproj_max_seq() -> int: return _i("CUSTOM_QPROJ_MAX_SEQ", 4) +def custom_qproj_tile() -> int: return _i("CUSTOM_QPROJ_TILE", 4) +def custom_fused_moe() -> bool: return _b("CUSTOM_FUSED_MOE", "0") +def custom_fused_moe_bits() -> int: return _i("CUSTOM_FUSED_MOE_BITS", 6) +def custom_fused_moe_lanes() -> int: return _i("CUSTOM_FUSED_MOE_LANES", 8) +def custom_fused_moe_block() -> int: return _i("CUSTOM_FUSED_MOE_BLOCK", 256) +def custom_fused_moe_max_seq() -> int: return _i("CUSTOM_FUSED_MOE_MAX_SEQ", 4) + + +# ============================ native MoE 后端 ============================ +def native_moe() -> bool: return _b("NATIVE_MOE", "0") +def native_moe_mlx_op() -> bool: return _b("NATIVE_MOE_MLX_OP", "0") +def native_moe_synthetic() -> bool: return _b("NATIVE_MOE_SYNTHETIC", "0") +def native_moe_slot_pool() -> bool: return _b("NATIVE_MOE_SLOT_POOL", "0") +def native_moe_raise() -> bool: return _b("NATIVE_MOE_RAISE", "0") +def native_moe_slot_cap() -> int: return _i("NATIVE_MOE_SLOT_CAP", 96) +def native_moe_stage_cache() -> bool: return _b("NATIVE_MOE_STAGE_CACHE", "1") +def native_moe_stage_cache_experts() -> int: return _i("NATIVE_MOE_STAGE_CACHE_EXPERTS", 96) +def native_moe_stage_bundle_cache() -> bool: return _b("NATIVE_MOE_STAGE_BUNDLE_CACHE", "1") +def native_moe_stage_cache_bundles() -> int: return _i("NATIVE_MOE_STAGE_CACHE_BUNDLES", 16) +def native_moe_stage_prefetch() -> bool: return _b("NATIVE_MOE_STAGE_PREFETCH", "0") + + +# ============================ 流式 blob ============================ +def stream_blob() -> bool: return _b("STREAM_BLOB", "0") +def stream_blob_loader() -> bool: return _b("STREAM_BLOB_LOADER", "0") +def stream_blob_bg() -> bool: return _b("STREAM_BLOB_BG", "0") +def stream_blob_workers() -> int: return _i("STREAM_BLOB_WORKERS", 8) +def stream_blob_window() -> int: return _i("STREAM_BLOB_WINDOW", 3) +def stream_blob_nocache(default: str = "1") -> bool: return _b("STREAM_BLOB_NOCACHE", default) +# 默认 12:双源侧区(cap32+侧区32)K=3 MTP 甜点扫描结果(REPEAT=2)。瓶颈是 SSD 读盘时序而非容量, +# 加大 budget 只会灌满共享读队列→预取到得更晚(timing miss↑)。W=24/B=12 在 hit(0.901)/disk(3874)/ +# tok·s(11.61)/确定性(nmis=2)全面优于旧 W=32/B=16(10.0 tok·s、nmis=43),tok·s +16%。 +def stream_blob_bg_budget(default: int = 12) -> int: return _i("STREAM_BLOB_BG_BUDGET", default) +# 按需 miss 批量并行读：把本层所有 miss 专家收成一批，一次 load_experts(8-worker 并行 pread)， +# 取代逐专家串行 pread（实测串行 6.4GB/s vs 并行 8 路 22GB/s）。仅影响读取调度，结果等价。 +# 默认开：剖析实测 decode 时间 ~92% 花在每层 host 回退路径，其中字节物化是大头；批量堆叠 +# (每段一次 mx.array 取代逐专家 6N 次构造)在 cap=12 把 decode 从 ~466ms 降到 ~408ms(~12%)。 +def batch_miss_read() -> bool: return _b("BATCH_MISS_READ", "1") +# demand miss 走 C++ 原生物化(load_experts_native)：blob_load 把字节 pread 直进 MLX 数组、 +# eval 时在 C++ 跑、绕 GIL、惰性。比 stacked 再快 ~6%(把物化从 acquire 挪到 eval)。 +# 默认关(opt-in)：实测它偶发把末尾 token 推进双稳态慢挡(2-3s/token 的"悬崖")， +# 其 lazy 物化推高内存压力,稳定性不如 stacked(纯 numpy 同步、4 次跑 0 慢挡)。仅在确认 +# 环境内存充裕、需要那 6% 时手动开。不消费 async prefetch buffer，故仅在 not async_prefetch 接入。 +def native_demand_loader() -> bool: return _b("NATIVE_DEMAND_LOADER", "0") +def staging_ring() -> int: return _i("STAGING_RING", 2) # 安全下限=2：MTP 每步 verify+replay 各对同层 submit 一次 +def staging_pread_parallel() -> bool: return _b("STAGING_PREAD_PARALLEL", "0") # staging fill 派后台池并行;默认关:实测不降 timing miss(IO 受限)且弱化 buffer 新鲜度不变量,详见 benchmarks/reports/staging-pread-parallel-2026-06-25.md +def stg_verify() -> bool: return _b("STG_VERIFY", "0") # 诊断:acquire_gpu 命中后池槽字节真值校验,默认关、对主路径零影响 +def stream_blob_prefetch_budget(default: int) -> int: return _i("STREAM_BLOB_PREFETCH_BUDGET", default) + + +# ============================ 跨层 / 预取 ============================ +# 注意 AHEAD 默认按调用方传：跨层 hook=0、native 预取=1。 +def cross_layer_prefetch() -> bool: return _b("CROSS_LAYER_PREFETCH", "0") +def cross_layer_ahead(default: int = 0) -> int: return _i("CROSS_LAYER_PREFETCH_AHEAD", default) +def cross_layer_mult() -> int: return max(1, _i("CROSS_LAYER_PREFETCH_MULT", 2)) +# 预测宽度（方案B）：预测 top-N 候选用于"减常驻"，N 大→recall 高且不占内存（只是 gate argpartition）。 +# 真正占 staging 的是过滤常驻后、按分截断到 staging budget 的缺口子集。 +# 默认 24:双源侧区 K=3 甜点扫描(REPEAT=2)。realized hit 在 W≥24 就到顶(~0.90),再加宽只抬"名义 +# recall"——多出的候选在 SSD 时序上落不进侧区且抢带宽,W=48→hit 0.85、W=64→0.75、tok·s 一路下滑。 +# 收窄到刚好覆盖可及工作集(W=24)最快最省。详见 benchmarks/reports/prefetch-width-budget-sweep-2026-07-01.md。 +def cross_layer_predict_width() -> int: return _i("CROSS_LAYER_PREDICT_WIDTH", 24) +# per-layer 自适应 ahead：早层用小 ahead 保召回、晚层用大 ahead 保时序（默认来自实测）。 +def cross_layer_cutoff() -> int: return _i("CROSS_LAYER_CUTOFF", 6) # 切点：层号 int: return _i("CROSS_LAYER_AHEAD_LO", 1) # 早层 ahead（保召回） +def cross_layer_ahead_hi() -> int: return _i("CROSS_LAYER_AHEAD_HI", 3) # 晚层 ahead（保时序） +def predict_use_x() -> bool: return _b("PREDICT_USE_X", "1") # 默认用本层 MoE 输入 x（更新鲜，+3.6pp recall）；=0 回退旧 norm 路径 +def predict_agg() -> str: return _s("PREDICT_AGG", "max") # K+1 token 聚合：max|mean|union +def predict_union_k() -> int: return _i("PREDICT_UNION_K", 8) # union 时每 token 取的 top-k（控候选数） +def native_fused_prefetch() -> bool: return _b("NATIVE_FUSED_PREFETCH", "0") +# 池侧区零拷贝双源(单/双缓冲)：opt-in、默认 off。VirtualPool 收口，消掉 promote 拷贝。 +# 侧区有两种淘汰策略(SIDEREGION_LFU 门控): +# - 旧"∉P 全清"(默认):侧区=一次性预取批,不积累→hit 仅 0.709(反低于基线 0.763)。 +# - 新"持久 LFU"(SIDEREGION_LFU=1,单代 spec_gens=1):跨步累积热专家,只读新增。 +# 实测(80B,cap=32,warmup64,见 report sideregion-lfu-2026-07-01): +# LFU spec=8 → hit 0.73 / active 4.76GB(省内存)；LFU spec=32 → hit 0.81 / 6.9GB(提命中,+8% tok/s)。 +# 命中在 ~0.81 饱和(加 warmup 无效),0.85+ 仍需真加常驻槽(cap=64→0.869)。 +# 注:dual on 各 spec 均有 run-to-run token 漂移(良性时序噪声,字节校验 0 BAD),故默认 off。 +def zerocopy_dual_source() -> bool: return _b("ZEROCOPY_DUAL_SOURCE") +def pool_spec_slots() -> int: return _i("POOL_SPEC_SLOTS", 3) # 每层侧区投机槽数(LFU 推荐 8 省内存 / 32 提命中) +def sideregion_lfu() -> bool: return _b("SIDEREGION_LFU", "1") # 侧区持久 LFU 单缓冲二级缓存(默认 on=生产路径);SIDEREGION_LFU=0 回退 legacy 双缓冲 +def native_no_submit() -> bool: return _b("NATIVE_NO_SUBMIT", "0") +def native_no_promote() -> bool: return _b("NATIVE_NO_PROMOTE", "0") +def native_materialize() -> bool: return _b("NATIVE_MATERIALIZE", "0") +def file_prefetch_global_budget() -> int: return _i("FILE_PREFETCH_GLOBAL_BUDGET", 64) +def stage_prefetch_global_budget() -> int: return _i("STAGE_PREFETCH_GLOBAL_BUDGET", 64) +def stage_prefetch_min_score() -> float: return _f("STAGE_PREFETCH_MIN_SCORE", 0) +def stage_prefetch_per_layer_budget(default: int = 4) -> int: return _i("STAGE_PREFETCH_PER_LAYER_BUDGET", default) + + +# ============================ 前缀快照(跨会话 system+tools 复用) ============================ +# 每个会话开头都是同一段长 system+tools 提示词(agentic 场景 ~2600 tok),新会话首轮 +# 要全量 prefill(实测 158s)。快照前 N 个 token 的 KV/递归态,后续同前缀会话直接跳过。 +# 0=关闭。注意:仅当 prompt 明显长于该值才启用(短提示词无收益)。 +def prefix_snapshot_head() -> int: return _i("PREFIX_SNAPSHOT_HEAD", 2048) +# 快照条目上限(LRU),每条约占几十 MB(线性层递归态为主) +def prefix_snapshot_entries() -> int: return _i("PREFIX_SNAPSHOT_ENTRIES", 4) + + +# ============================ 工具白名单 ============================ +# 渲染进 chat template 的工具白名单(按工具名),用于把 prompt 头部钉成固定的一段。 +# +# 快照命中的前提是头部**字节级**相同(key = ids[:head] 的哈希)。而调用方的工具发现是动态的: +# 同一段 system 提示词会配不同的工具集发来,头部随之变化,快照必然 miss —— 每个新会话都要 +# 全量 prefill 4000 token 上下。实测本部署已因此存下两份不同头部的快照(3972 / 4261 token), +# 互不命中。白名单把工具集收敛成固定集合后,头部才稳定、快照才能真正消灭"首轮等待"。 +# 省 token 只是附带好处(工具定义约占头部三分之一)。 +# +# 默认只放画像(profile)子系统实际在用的 4 个工具;其余(如排班 metro_*)本部署不处理。 +# 设为空(TOOLS_ALLOW=)即关闭过滤、原样透传全部工具。 +_TOOLS_ALLOW_DEFAULT = ("profile_list_employees,profile_get_portrait_overview," + "profile_get_person_report,profile_get_team_report") + + +def tools_allow() -> "list[str]": + spec = _s("TOOLS_ALLOW", _TOOLS_ALLOW_DEFAULT) + return [s for s in (part.strip() for part in spec.split(",")) if s] + + +# ============================ MTP ============================ +def mtp_verify_mode() -> str: return _s("MTP_VERIFY_MODE", "batch") +# MTP drafter 量化位宽(load_mtp bits 参数):4(默认,省内存)| 8(草稿更准,接受率更高, +# 实测杠杆:accept_len 提升即等比例提速;代价 ~0.7GB 内存) +def mtp_bits() -> int: return _i("MTP_BITS", 4) +# 喂给 MTP drafter 的主模型 hidden:pre_norm(默认,与训练/验证一致,接受率高)| post_norm(旧行为) +def mtp_hidden() -> str: return _s("MTP_HIDDEN", "pre_norm") +# 分块 prefill 的每块 token 数:整段 prefill 一次前向的激活峰值 ∝ prompt 长度(每 MoE 层瞬时 +# 物化大量唯一专家 + 长序列激活),把 prompt 切成小块逐块喂入(KV/SSM 按 offset 因果累积、末块 +# 末位 logits 与整段等价),峰值压回 ∝chunk,使 prefill 与 decode 同稳态。默认 2(沿用 DeepSeek +# 实证:每块唯一专家少、稳走 resident 池便宜路径);PREFILL_CHUNK=0 关闭分块(整段 prefill)。 +def prefill_chunk() -> int: return _i("PREFILL_CHUNK", 2) + + +# ============================ KV 量化(IsoQuant K4/V3)============================ +# 只作用于 12 个全注意力层。SO(4) 块旋转去相关 + 非对称 K4/V3 仿射量化:128k KV 3.0→~0.68 GiB。 +# 线性层(Gated DeltaNet 递归态)不动。质量验收:token 一致率≥95% + logits cosine≥0.99。 +def kv_quant() -> bool: return _b("KV_QUANT", "0") +def kv_k_bits() -> int: return _i("KV_K_BITS", 4) # Key 位宽(默认 4) +def kv_v_bits() -> int: return _i("KV_V_BITS", 3) # Value 位宽(默认 3,非对称) +def kv_group_size() -> int: return _i("KV_GROUP_SIZE", 64) # 仿射量化分组(整除 head_dim=256) +def kv_rotate() -> bool: return _b("KV_ROTATE", "1") # SO(4) 块旋转去相关(默认开) +def kv_rot_seed() -> int: return _i("KV_ROT_SEED", 0) # 旋转随机种子(data-oblivious、固定) + + +# ============================ profiling / 诊断 ============================ +def window_prof() -> bool: return _b("WINDOW_PROF", "0") +def predict_recall_prof() -> bool: return _b("PREDICT_RECALL_PROF", "0") +def miss_attrib() -> bool: return _b("MISS_ATTRIB", "0") # miss 归因：A(预测到没到位)/B(没预测到) +def route_trace_enabled() -> bool: return _b("ROUTE_TRACE", "0") +def stream_prof() -> bool: return _b("STREAM_PROF", "0") +def probe_predict_only() -> bool: return _b("PROBE_PREDICT_ONLY", "0") +def probe_perlayer_sync() -> bool: return _b("PROBE_PERLAYER_SYNC", "0") +# 预取 host 墙钟探针：量 predict/submit/promote 各段主线程不可重叠的 CPU 时间。默认关、零开销。 +def prefetch_tprof() -> bool: return _b("PREFETCH_TPROF", "0") +# 并集专家数探针:按前向 seq 分桶记每层路由专家并集大小(seq=K 即 MTP verify 的专家并集)。默认关。 +def union_prof() -> bool: return _b("UNION_PROF", "0") +# 接受率 top-k 覆盖探针(>0 且 profile 时):记每个草稿位置 MTP 的 top-k 候选,量模型真实 token +# 是否落在 top-2/top-3 里 = 树形展开的救回上界。默认 0=关。 +def accept_topk() -> int: return _i("ACCEPT_TOPK", 0) +# 最小树:位置1 展开 top-2。仅当 A 链首草稿被拒且 B 候选=真实 token 时,额外跑一次 B 链前向救回。 +# 两分支各为独立 batch=1 seq=K 前向(线性层不能批处理树,故不拍平),per-forward union 不变。默认关。 +def tree_top2() -> bool: return _b("TREE_TOP2", "0") +# 第2 草稿位置(pos1)top-2 救回:在 tree_top2 基础上,额外抽 chainC(第2 位次选分支),当第1 位 +# 命中但第2 位被拒、且 chainC 第2 token=模型真值时改验 chainC。探针实测 pos1「首选错次选对」比例 +# (~11%)高于 pos0(~7%),消融证实接受长度确实 +2.74%(bit-lossless);但每次救回要多一次主模型 +# 前向,该成本在当前硬件/批量下恰好抵消 token 收益,净 tok/s 无提升(见 benchmarks/_bench_p1_ablation)。 +# 故默认关,保护已验证的 pos0 纯路径 tok/s 收益;留作前向变廉价/批量变大时收益翻正的储备,一行 env 开。 +def tree_top2_p1() -> bool: return _b("TREE_TOP2_P1", "0") +# 完整树形验证(batch-of-paths):把 P 条候选路径拍到 batch 维,一次 batched 前向并行验证所有路径, +# 选接受最长的路径提交(提取赢家 row)。每条路径是普通线性序列,故线性层/全注意力层都走成熟的 +# batch 前向(无需改 kernel);单次前向的 batch=P 计算加宽了预取窗口,同时多路径提升接受长度 +# ——一举拿下 accept_len 与 hit_rate 两只鸟。默认关(与 tree_top2/普通链验证互斥,优先级最高)。 +def tree_verify() -> bool: return _b("TREE_VERIFY", "0") +# 树分支数(候选路径条数 P)。当前 drafter 在位置1 展开 top-P,P=2 即 top-2。 +def tree_branches() -> int: return _i("TREE_BRANCHES", 2) +# 置信度门控动态深度(P-MTP 风格):逐位抽草稿时累计置信度 C_i=p0·…·p_i,C_i≥tau 且未到 depth_max +# 就继续加深,否则本步只 verify 到当前深度。低置信步收缩(省专家加载/cap 压力),高置信步至多到 +# depth_max。探针实测置信度对接受率区分度极强(高低置信接受率差 +42~59pp)。消融结论(见 +# benchmarks/reports,_bench_adaptive):收益全部来自"向下收缩",τ=0.3、depth_max=3 即 +5~6% tok/s +# 且 bit-lossless、零额外显存;向上扩到 K=4 反而更慢(第4 位专家加载成本 > 多接受的 token),且在 +# 生产 EXPERT_SLOTS=32 下 seq=4·top_k=40>cap 会溢出致有损。故 depth_max 默认 3(=基础 K,slots=32 安全), +# 要扩到 4 必须同时把 EXPERT_SLOTS 提到 ≥40。默认关,作可选加速路径(与 tree_top2 互斥)。 +def adaptive_depth() -> bool: return _b("MTP_ADAPTIVE_DEPTH", "0") +def conf_tau() -> float: return _f("MTP_CONF_TAU", 0.3) +def depth_max() -> int: return _i("MTP_DEPTH_MAX", 3) +# 合并路径:在动态深度基础上,对"被保留成深链(n>=2)"的步叠加 pos0 top-2 救回(首 token 被拒且 +# top-2=模型真值时改验 B 链)。两机理正交(动态深度压每步成本、救回抬接受长度),但都作用于低置信步、 +# 方向相反(depth=1 的浅步无位置可救),故叠加非简单相加,须实测。默认关。 +def adaptive_rescue() -> bool: return _b("MTP_ADAPTIVE_RESCUE", "0") + + +def parse_layers_env(name: str) -> "set[int] | None": + """解析 "0-3,5,8" 形式的层集合环境变量；为空返回 None（表示全部层）。""" + spec = os.environ.get(name, "").strip() + if not spec: + return None + out: "set[int]" = set() + for part in spec.split(","): + part = part.strip() + if not part: + continue + if "-" in part: + a, b = part.split("-", 1) + out.update(range(int(a), int(b) + 1)) + else: + out.add(int(part)) + return out diff --git a/mlx_streaming/core/__init__.py b/mlx_streaming/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mlx_streaming/core/cache/__init__.py b/mlx_streaming/core/cache/__init__.py new file mode 100644 index 0000000..e316e52 --- /dev/null +++ b/mlx_streaming/core/cache/__init__.py @@ -0,0 +1 @@ +"""专家缓存模块：常驻池(resident_pool) / LRU+文件后端存储(expert_store) / blob 流式源(blob_loader)。""" diff --git a/mlx_streaming/core/cache/autopin.py b/mlx_streaming/core/cache/autopin.py new file mode 100644 index 0000000..146813f --- /dev/null +++ b/mlx_streaming/core/cache/autopin.py @@ -0,0 +1,174 @@ +"""AUTOPIN：专家热度持久化 + 启动预热钉死。 + +两部分： +- **热度计数与持久化**：搭已有的物化点零同步计频——block host 路径的 flat、 + acquire_gpu 的 LFU piggyback(全命中)与 miss 回退 flat；dual decode/verify 无 Python + 物化点，由 C++ g_real 的 LFU freq 在落盘时按增量差分合并。节流写 + {EXPERT_DIR}/pool_usage.json(同目录临时文件 + rename 防半写；读不到/损坏静默从零)。 +- **启动预热钉死**：build_streaming_model 末尾按历史热度把每层 top-N 热专家 + (N = floor(cap_for × AUTOPIN_BUDGET_FRAC))经 blob 并行读预填进常驻池并 pin 住 + (不参与任何驱逐)，消灭冷启动慢热。 + +覆盖率口径：prefill/host 路径与 GPU remap miss 回退全计；非 dual 的 decode 全命中 +快路径搭 LFU piggyback 计(inds 已物化，零额外同步)；dual decode/verify 靠 C++ freq +兜底(仅 lfu 策略计频)。EVICT_POLICY≠lfu 时两条 GPU 快路径均不计——绝不为计数给 +decode 热路径新增 GPU→host 同步。 +""" +import atexit +import json +import os +import tempfile +from collections import Counter +from typing import Dict, List + +from mlx_streaming import config + +_USAGE_NAME = "pool_usage.json" + + +def usage_path() -> str: + """热度文件路径：AUTOPIN_USAGE_FILE 覆盖，缺省 {EXPERT_DIR}/pool_usage.json。""" + p = config.autopin_usage_file() + return p or os.path.join(config.expert_dir(), _USAGE_NAME) + + +def load_usage(path: str) -> "Dict[int, Counter]": + """读热度文件；不存在/损坏/结构非法一律静默返回空(从零开始)。""" + try: + with open(path) as f: + raw = json.load(f) + return {int(l): Counter({int(e): int(c) for e, c in d.items()}) + for l, d in raw.items()} + except Exception: + return {} + + +def save_usage(path: str, counts: "Dict[int, Counter]") -> None: + """原子写：先写同目录临时文件再 rename，防半写损坏。失败静默(不阻断推理)。""" + try: + d = os.path.dirname(path) or "." + fd, tmp = tempfile.mkstemp(dir=d, prefix=".pool_usage.", suffix=".tmp") + with os.fdopen(fd, "w") as f: + json.dump({str(l): {str(e): c for e, c in cnt.items()} + for l, cnt in counts.items()}, f) + os.replace(tmp, path) + except Exception: + pass + + +class HeatCounter: + """进程内热度累计器：note() 增量计频，节流 + atexit 落盘。 + + 前向边界用「MoE 层号回绕」近似(与 VirtualPool.begin_forward 同判据)：MoE 块按层号 + 递增被调，出现 layer <= 上次即新前向。路径在创建时快照，避免进程退出时 env 已变 + 导致 atexit 写错位置。 + """ + + def __init__(self, path: str): + self.path = path + self.counts: "Dict[int, Counter]" = {} + self._last_layer = -1 + self._forwards_since_save = 0 + # C++ 累计频次的已合并基线(real_freq_dump 是进程累计值，按差分合并防重复计) + self._cpp_baseline: "Dict[int, Dict[int, int]]" = {} + + def note(self, layer: int, ids: List[int]) -> None: + layer = int(layer) + if layer <= self._last_layer: + self._forwards_since_save += 1 + self._last_layer = layer + c = self.counts.get(layer) + if c is None: + c = self.counts[layer] = Counter() + for e in ids: + c[int(e)] += 1 + if self._forwards_since_save >= config.autopin_save_every(): + self.flush() + + def tick(self) -> None: + """显式前向边界(dual decode 无 Python 计频点，由 begin_forward 驱动周期落盘)。""" + self._forwards_since_save += 1 + if self._forwards_since_save >= config.autopin_save_every(): + self.flush() + + def merge_cpp_freq(self) -> None: + """把 dual decode 的 C++ LFU 累计频次按增量差分合并进来。""" + try: + import mlx_streaming.native_moe_ext as _N + flat = _N.real_freq_dump() + except Exception: + return + cur: "Dict[int, Dict[int, int]]" = {} + for i in range(0, len(flat), 3): + l, e, c = int(flat[i]), int(flat[i + 1]), int(flat[i + 2]) + cur.setdefault(l, {})[e] = c + for l, em in cur.items(): + base = self._cpp_baseline.get(l, {}) + cnt = self.counts.setdefault(l, Counter()) + for e, c in em.items(): + b = base.get(e, 0) + cnt[e] += c - b if c >= b else c # real_reset 后 c None: + self.merge_cpp_freq() + save_usage(self.path, self.counts) + self._forwards_since_save = 0 + + +_counter: "HeatCounter | None" = None + + +def _get_counter() -> HeatCounter: + global _counter + if _counter is None: + _counter = HeatCounter(usage_path()) + atexit.register(_counter.flush) + return _counter + + +def note(layer: int, ids: List[int]) -> None: + """路由热度计数入口(热路径调用)：AUTOPIN=0 立即返回，零行为。""" + if not config.autopin(): + return + _get_counter().note(layer, ids) + + +def tick() -> None: + """前向边界入口(virtual_pool begin_forward 调用)：AUTOPIN=0 立即返回。""" + if not config.autopin(): + return + _get_counter().tick() + + +def counter() -> "HeatCounter | None": + """测试/诊断用：当前计数器(未触发过计数则为 None)。""" + return _counter + + +def warm_start_pin(store) -> "dict": + """启动预热钉死：按 usage 文件每层取 top-N 热专家，经 store.pin_batch 预填进池并 pin 住。 + + N = floor(该层 cap_for × AUTOPIN_BUDGET_FRAC)。pin 加载由 pin_batch 收口：有 blob_loader + 走并行 pread(生产唯一数据源)，无则退回逐专家 per-expert 文件(兼容旧部署)。 + usage 缺失/为空 → 返回零摘要(后台计数仍在累计，供下次启动用)；单层失败不阻断整体。 + """ + summary = {"layers": 0, "pinned": 0, "skipped_layers": 0, "seconds": 0.0} + usage = load_usage(usage_path()) + if not usage: + return summary + import time + t0 = time.perf_counter() + frac = config.autopin_budget_frac() + for layer in sorted(usage): + n = int(store.cap_for(layer) * frac) + if n <= 0: + continue + top = [e for e, _ in usage[layer].most_common(n)] + try: + summary["pinned"] += store.pin_batch(layer, top) + summary["layers"] += 1 + except Exception: + summary["skipped_layers"] += 1 + summary["seconds"] = round(time.perf_counter() - t0, 2) + return summary diff --git a/mlx_streaming/core/cache/blob_loader.py b/mlx_streaming/core/cache/blob_loader.py new file mode 100644 index 0000000..8436c05 --- /dev/null +++ b/mlx_streaming/core/cache/blob_loader.py @@ -0,0 +1,238 @@ +"""全流式 blob 专家源：并行 pread + F_NOCACHE + 即时物化为 MLX 量化数组。 + +字节布局由 prep/blob_layout.py 统一描述（单一真相源），支持两种格式： +- v1 affine（expert_blob_v1）：每 proj 段序 [weight, scales, biases]；weight uint32， + scales/biases 存 uint16 原始位、用时 .view(bfloat16)。 +- v2 mxfp4（expert_blob_v2_mxfp4）：每 proj 段序 [weight, scales]（无 biases）； + weight uint32，scales 为 uint8 原始位、绝不 view(bf16)。 +计算复用 MLX quantized_matmul / gather_qmm（不碰已 NO-GO 的 fused kernel）。 +""" +import os +import threading +from collections import OrderedDict +from concurrent.futures import ThreadPoolExecutor + +import mlx.core as mx +import numpy as np + +from mlx_streaming import config + +# macOS fcntl：F_NOCACHE 提示内核不要把读过的页留在 page cache（支撑低内存目标）。 +_F_NOCACHE = 48 + + +class BlobExpertSource: + def __init__(self, blob_dir: str, hidden: int, inter: int, group: int, bits: int, + num_experts: int, workers: int = 8, nocache: bool = True, + quant_mode: str = "affine", blob_format: "str | None" = None): + self.dir = blob_dir + self.h, self.i, self.g, self.b, self.ne = hidden, inter, group, bits, num_experts + self.workers = workers + self.nocache = nocache + # blob 格式：mxfp4 默认 v2（无 biases、scales uint8），其余默认 v1 affine。 + self.quant_mode = quant_mode + self.blob_format = blob_format or ( + "expert_blob_v2_mxfp4" if quant_mode == "mxfp4" else "expert_blob_v1") + self._segs, self.stride = self._layout() + self._fds: "dict[int, int]" = {} + # 后台预取：只预读原始字节进 _pf_cache（不建 mx.array，可后台线程跑）； + # 滚动窗口只保留最近 window 层的预取字节，支撑低内存。 + self.window = config.stream_blob_window() + self._pf_pool = ThreadPoolExecutor(max_workers=workers) + self._pf_cache: "OrderedDict[tuple, bytes]" = OrderedDict() + self._pf_futures: "dict[tuple, object]" = {} + self._pf_layers: "list[int]" = [] + self._lock = threading.Lock() + self.preads = 0 + self.prefetch_hits = 0 + + def _layout(self): + # 段表统一由 blob_layout.layout_for 给出（dtype 为字符串 "uint32"/"uint16"/"uint8"）。 + from mlx_streaming.prep.blob_layout import layout_for + return layout_for(self.blob_format, self.h, self.i, self.b, self.g) + + def _fd(self, layer: int) -> int: + fd = self._fds.get(layer) + if fd is None: + fd = os.open(os.path.join(self.dir, f"layer{layer:02d}.blob"), os.O_RDONLY) + if self.nocache: + try: + import fcntl + fcntl.fcntl(fd, _F_NOCACHE, 1) + except (OSError, ValueError): + pass + self._fds[layer] = fd + return fd + + def _materialize(self, raw: bytes, view_bf16: bool = True) -> dict: + # view_bf16=False：scales/biases 保留 uint16（.view(bfloat16) 在后台线程会报 + # no Stream(gpu,0)，故后台物化时关掉，由主线程消费时再 view）。 + # 段表 dtype 为字符串：仅 v1 affine 的 uint16 scales/biases 是 bf16 位重解释； + # mxfp4 的 uint8 scales 保持原样，绝不 view(bf16)。 + out = {} + off = 0 + for proj, tensor, dt, shape, nb in self._segs: + np_dt = np.dtype(dt) + view = np.frombuffer(raw, dtype=np_dt, count=nb // np_dt.itemsize, offset=off).reshape(shape) + arr = mx.array(view) + if view_bf16 and dt == "uint16" and tensor in ("scales", "biases"): + arr = arr.view(mx.bfloat16) + out[f"{proj}.{tensor}"] = arr + off += nb + return out + + def _pread(self, layer: int, e: int) -> bytes: + return os.pread(self._fd(layer), self.stride, e * self.stride) + + def prefetch_async(self, layer: int, expert_ids) -> None: + """后台并行预读预测专家的字节（仅 IO，不建 mx.array）。供跨层预测调用。""" + for e in (int(x) for x in expert_ids): + key = (layer, e) + with self._lock: + if key in self._pf_cache or key in self._pf_futures: + continue + self._pf_futures[key] = self._pf_pool.submit(self._pf_job, layer, e) + self._evict_window(layer) + + def _pf_job(self, layer: int, e: int) -> bytes: + raw = self._pread(layer, e) + with self._lock: + self._pf_cache[(layer, e)] = raw + self._pf_futures.pop((layer, e), None) + return raw + + def _evict_window(self, layer: int) -> None: + with self._lock: + if layer not in self._pf_layers: + self._pf_layers.append(layer) + while len(self._pf_layers) > self.window: + old = self._pf_layers.pop(0) + for key in [k for k in self._pf_cache if k[0] == old]: + del self._pf_cache[key] + + def wait_prefetch(self) -> None: + with self._lock: + futs = list(self._pf_futures.values()) + for f in futs: + f.result() + + def read_raw(self, layer: int, expert_ids) -> "list[bytes]": + """取原始字节：优先用预取缓存/在途结果，未命中的并行 pread（仅 IO）。""" + ids = [int(e) for e in expert_ids] + results: "dict[int, bytes]" = {} + misses = [] + for e in ids: + key = (layer, e) + with self._lock: + if key in self._pf_cache: + results[e] = self._pf_cache[key] + self.prefetch_hits += 1 + continue + fut = self._pf_futures.get(key) + if fut is not None: + results[e] = fut.result() + self.prefetch_hits += 1 + else: + misses.append(e) + + def rd(e): + return e, self._pread(layer, e) + + if misses: + self.preads += len(misses) + if self.workers <= 1 or len(misses) <= 1: + for e in misses: + results[e] = self._pread(layer, e) + else: + # 复用持久线程池（之前每次调用新建/销毁 ThreadPoolExecutor → 大量线程创建 + GIL 争用）。 + for e, raw in self._pf_pool.map(rd, misses): + results[e] = raw + return [results[e] for e in ids] + + def load_experts(self, layer: int, expert_ids, view_bf16: bool = True) -> dict: + """返回 {expert_id: {proj.tensor: mx.array}}。并行读字节，lazy 物化 mx.array。 + + view_bf16=False：scales/biases 留 uint16（供后台线程用，避免 .view 报 no Stream）。 + 注：零拷贝(preadv 进 MLX buffer)实测在真实 lazy-eval 流里更慢——为拿可写 buffer + 需 per-expert mx.eval 同步，代价高于省掉的拷贝(见报告)。故保留 lazy frombuffer 路径。 + """ + ids = [int(e) for e in expert_ids] + raws = self.read_raw(layer, ids) + return {e: self._materialize(raw, view_bf16=view_bf16) for e, raw in zip(ids, raws)} + + def load_experts_stacked(self, layer: int, expert_ids, view_bf16: bool = True) -> dict: + """批量物化：每段只建一个 (N,*shape) mx.array（np.stack 后单次构造）， + 取代逐专家 6N 次 frombuffer+mx.array。返回 {proj.tensor: (N,*shape)}，按 ids 顺序。 + + 与 load_experts 同源（同段表、同 view_bf16 语义），只是把"逐专家逐段"折成 + "逐段一次堆叠"——demand 多 miss 消费的碎 mx.array 构造从 6N 降到 6。 + """ + ids = [int(e) for e in expert_ids] + raws = self.read_raw(layer, ids) + out = {} + off = 0 + for proj, tensor, dt, shape, nb in self._segs: + np_dt = np.dtype(dt) + cnt = nb // np_dt.itemsize + views = [np.frombuffer(raw, dtype=np_dt, count=cnt, offset=off).reshape(shape) + for raw in raws] + arr = mx.array(np.stack(views, axis=0)) # (N, *shape) 单次构造 + if view_bf16 and dt == "uint16" and tensor in ("scales", "biases"): + arr = arr.view(mx.bfloat16) + out[f"{proj}.{tensor}"] = arr + off += nb + return out + + def load_experts_native(self, layer: int, expert_ids, view_bf16: bool = False) -> dict: + """native 物化：用 C++ blob_load 把字节 pread 进 MLX 数组（eval 时在 C++ 跑、绕 GIL）， + Python 侧只建惰性切片/view 图（无数据拷贝）。目的是把"拷贝"挪出 GIL，减少后台线程 + 对主线程 per-layer 派发的争用。 + """ + import mlx_streaming.native_moe_ext as _N + ids = [int(e) for e in expert_ids] + path = os.path.join(self.dir, f"layer{layer:02d}.blob") + raw = _N.blob_load(path, mx.array(ids, dtype=mx.uint32), self.stride) # [n, stride] uint8（lazy） + out = {} + for i, e in enumerate(ids): + row = raw[i] + d = {} + off = 0 + for proj, tensor, dt, shape, nb in self._segs: + seg = row[off:off + nb] + # 按段 dtype 重解释：weight→uint32；v1 affine 的 uint16 scales/biases + # 可再 view(bf16)；mxfp4 的 uint8 scales 保持 uint8。 + if dt == "uint32": + arr = seg.view(mx.uint32).reshape(shape) + elif dt == "uint16": + arr = seg.view(mx.uint16).reshape(shape) + if view_bf16: + arr = arr.view(mx.bfloat16) + else: # uint8（mxfp4 scales） + arr = seg.reshape(shape) + d[f"{proj}.{tensor}"] = arr + off += nb + out[e] = d + return out + + def keys(self) -> "list[str]": + return [f"{proj}.{tensor}" for proj, tensor, _, _, _ in self._segs] + + def acquire(self, layer: int, expert_ids): + """对齐 ResidentExpertPool.acquire：返回 (pool_arrays, slots)。 + + pool_arrays: {proj.tensor: (n_uniq,...) 堆叠数组}，喂给 _sub.forward。 + slots: 每个 expert_id 在 unique 列表中的下标（= local，reshape 前）。 + """ + ids = [int(e) for e in expert_ids] + uniq = list(dict.fromkeys(ids)) + experts = self.load_experts(layer, uniq) + pool = {k: mx.stack([experts[e][k] for e in uniq], axis=0) for k in self.keys()} + pos = {e: i for i, e in enumerate(uniq)} + slots = [pos[e] for e in ids] + return pool, slots + + def close(self): + self._pf_pool.shutdown(wait=True) + for fd in self._fds.values(): + os.close(fd) + self._fds.clear() diff --git a/mlx_streaming/core/cache/expert_store.py b/mlx_streaming/core/cache/expert_store.py new file mode 100644 index 0000000..45b0781 --- /dev/null +++ b/mlx_streaming/core/cache/expert_store.py @@ -0,0 +1,604 @@ +"""路线 B 专家后端：每层独立 LRU 缓存 + 按需切片，统计命中率。 + +与早期版本的两处提速改动： +- **每层独立 LRU**：每个 MoE 层各自一个 LRU，容量 = 每层槽数。解码自回归时相邻 + token 在同一层常路由到重叠专家，per-layer 隔离能显著抬高命中率（不再被别层挤掉）。 +- **去掉每次驱逐的 clear_cache 抖动**：驱逐只丢引用，靠 MLX 自带缓冲复用 + 外部 + set_cache_limit 控制上限，避免低命中率下疯狂 malloc/free。 + +stacked[layer] = {"weight": (E,O,I) [, "scales", "biases"]}（惰性，未 eval）。 +fetch(layer, expert_ids) 只取这几个专家、堆叠成 (k, ...) 返回。 +""" +import json +import os +import struct +from collections import OrderedDict, Counter +from typing import Dict, List + +import mlx.core as mx +import numpy as np + +from mlx_streaming import config +from mlx_streaming.core.mem import clear_cache +from mlx_streaming.core.cache.resident_pool import ResidentExpertPool, _POOL_INIT_SLOTS # noqa: F401 + +# macOS fcntl：F_NOCACHE 提示内核不要把读过的页留在 page cache。 +_F_NOCACHE = 48 +# safetensors dtype -> (numpy 读入类型, 需 view 的 mlx 类型)。BF16 numpy 无原生类型, +# 先读 uint16 原始位再 .view(bfloat16)。其余类型按需补。 +_ST_DTYPE = { + "U32": (np.uint32, None), + "U16": (np.uint16, None), + "I32": (np.int32, None), + "F32": (np.float32, None), + "F16": (np.float16, None), + "BF16": (np.uint16, mx.bfloat16), +} + + +def _pread_all(fd: int, size: int) -> bytes: + """把整文件读出(os.pread 可能短读,循环补齐)。""" + chunks = [] + off = 0 + while off < size: + b = os.pread(fd, size - off, off) + if not b: + break + chunks.append(b) + off += len(b) + return b"".join(chunks) + + +def load_safetensors_nocache(path: str) -> "Dict[str, mx.array]": + """用 F_NOCACHE 读 safetensors,返回与 mx.load 等价的 {name: mx.array}。 + + 绕过 OS page cache:每次都真实读 NVMe,基准结果可复现(不被页缓存冷热污染)。 + 与 mx.load 的数值/dtype 完全一致(U32 权重、BF16 scales/biases)。 + """ + fd = os.open(path, os.O_RDONLY) + try: + try: + import fcntl + fcntl.fcntl(fd, _F_NOCACHE, 1) + except (OSError, ValueError, ImportError): + pass + size = os.fstat(fd).st_size + data = _pread_all(fd, size) + finally: + os.close(fd) + n = struct.unpack(" "OrderedDict[int, Dict[str, mx.array]]": + c = self._caches.get(layer) + if c is None: + c = OrderedDict() + self._caches[layer] = c + return c + + def resident_count(self) -> int: + return sum(len(c) for c in self._caches.values()) + + def get_or_load(self, layer: int, e: int, loader) -> Dict[str, mx.array]: + cache = self._layer_cache(layer) + if e in cache: + self.hits += 1 + cache.move_to_end(e) + else: + self.misses += 1 + cache[e] = loader(layer, e) + cache.move_to_end(e) + while len(cache) > self.capacity: + cache.popitem(last=False) # 仅丢引用；不在热路径 clear_cache + if self.clear_on_evict: + clear_cache() + return cache[e] + + def hit_rate(self) -> float: + tot = self.hits + self.misses + return self.hits / tot if tot else 0.0 + + +def _stack_picked(picked: List[Dict[str, mx.array]]) -> Dict[str, mx.array]: + keys = picked[0].keys() + return {k: mx.stack([p[k] for p in picked]) for k in keys} + + +def _stack_cache_size(size: int | None) -> int: + if size is not None: + return max(0, int(size)) + return max(0, config.expert_stack_cache()) + + +def _stack_cache_get(cache, size: int, layer: int, key: tuple[int, ...]): + if size <= 0: + return None + layer_cache = cache.get(layer) + if layer_cache is None or key not in layer_cache: + return None + layer_cache.move_to_end(key) + return layer_cache[key] + + +def _stack_cache_put(cache, size: int, layer: int, key: tuple[int, ...], + value: Dict[str, mx.array]) -> None: + if size <= 0: + return + layer_cache = cache.get(layer) + if layer_cache is None: + layer_cache = OrderedDict() + cache[layer] = layer_cache + layer_cache[key] = value + layer_cache.move_to_end(key) + while len(layer_cache) > size: + layer_cache.popitem(last=False) + + +class LruExpertStore: + """内存堆叠后端：从常驻（惰性）堆叠张量按专家切片。容量为「每层槽数」。""" + + def __init__(self, stacked: Dict[int, Dict[str, mx.array]], capacity: int, + clear_on_evict: bool = False, stack_cache_size: int | None = None): + self._stacked = stacked + self.capacity = capacity + self._lru = _PerLayerLru(capacity, clear_on_evict) + self.stack_cache_size = _stack_cache_size(stack_cache_size) + self._stack_cache: "Dict[int, OrderedDict[tuple[int, ...], Dict[str, mx.array]]]" = {} + @property + def hits(self) -> int: + return self._lru.hits + + @property + def misses(self) -> int: + return self._lru.misses + + def resident_count(self) -> int: + return self._lru.resident_count() + + def cap_for(self, layer: int) -> int: + return self.capacity + + def _load_one(self, layer: int, e: int) -> Dict[str, mx.array]: + src = self._stacked[layer] + out = {} + for k, arr in src.items(): + sub = arr[e] # 切第 e 个专家 + mx.eval(sub) # 显式物化这一个专家 + out[k] = sub + return out + + def fetch(self, layer: int, expert_ids: List[int]) -> Dict[str, mx.array]: + key = tuple(int(e) for e in expert_ids) + cached = _stack_cache_get(self._stack_cache, self.stack_cache_size, layer, key) + if cached is not None: + for e in key: + self._lru.get_or_load(layer, e, self._load_one) + return cached + picked = [self._lru.get_or_load(layer, int(e), self._load_one) for e in expert_ids] + stacked = _stack_picked(picked) + _stack_cache_put(self._stack_cache, self.stack_cache_size, layer, key, stacked) + return stacked + + def hit_rate(self) -> float: + return self._lru.hit_rate() + + +class FileExpertStore: + """文件后端专家缓存：从离线拆分的 per-expert safetensors 按需加载，每层独立 LRU。 + + 文件命名：{root}/layer{layer:02d}_expert{e:03d}.safetensors， + 内容为扁平 dict，键形如 "gate_proj.weight"/"gate_proj.scales"/"up_proj.weight"... + capacity 语义为「每层槽数」（worst-case 常驻 ≈ capacity × MoE 层数）。 + """ + + def __init__(self, root: str, capacity: int, clear_on_evict: bool = False, + record: bool = False, stack_cache_size: int | None = None, + layer_caps: "Dict[int, int] | None" = None): + self.root = root + self.capacity = capacity + self._lru = _PerLayerLru(capacity, clear_on_evict) + self.stack_cache_size = _stack_cache_size(stack_cache_size) + self._stack_cache: "Dict[int, OrderedDict[tuple[int, ...], Dict[str, mx.array]]]" = {} + # ③ 热专家常驻：钉住的专家永不进 LRU、永不驱逐 + self._pinned: "Dict[int, Dict[int, Dict[str, mx.array]]]" = {} + self.pinned_hits = 0 + # 可选 per-layer bundle：把同层所有 per-expert safetensors 合并成一个文件， + # 运行时每层只 mx.load 一次，减少大量小文件 open/header parse 开销。 + self.use_bundles = config.expert_bundle() + self.bundle_dir = config.expert_bundle_dir(os.path.join(root, "layer_bundles")) + self.bundle_cache_size = config.expert_bundle_cache() + self._bundle_cache: "OrderedDict[int, Dict[str, mx.array]]" = OrderedDict() + self._bundle_key_cache: "Dict[int, Dict[int, list[tuple[str, str]]]]" = {} + self.bundle_loads = 0 + # 连续常驻池后端（acquire 路径用），与 LRU stack 路径共享 _load_one + # layer_caps：每层独立池容量(profile 驱动的无损省内存)，缺省层用全局 capacity + # demand miss 批量加载接线(三选一,acquire 优先 stacked > batch > 逐专家): + # - native(默认,且非 async_prefetch):C++ blob_load 直进 MLX 数组、绕 GIL、惰性,返回 + # 逐专家 {e:{k}} → 走 _place_experts。native 不消费 async prefetch buffer,故 gate 在 + # not async_prefetch(async 路径改用 _batch_load_resident,它会先吃 buffer)。 + # - batch_miss_read(默认):numpy 批量堆叠,每段一次 mx.array(取代逐专家 6N 次构造)。 + # - 都关:逐专家 loader。 + if config.native_demand_loader() and not config.async_prefetch(): + _batch_loader = self._batch_load_native + _stacked_loader = None + elif config.batch_miss_read(): + _batch_loader = self._batch_load_resident + _stacked_loader = (self._batch_load_stacked + if not config.async_prefetch() else None) + else: + _batch_loader = None + _stacked_loader = None + self._resident = ResidentExpertPool( + capacity, loader=self._load_one_resident, layer_caps=layer_caps, + batch_loader=_batch_loader, stacked_batch_loader=_stacked_loader) + # 激活频率统计（校准阶段用） + self.record = record + self._counts: "Dict[int, Counter]" = {} + self.async_prefetch = config.async_prefetch() + self.prefetch_buffer_size = config.prefetch_buffer_experts() + self._prefetch_buffer: "OrderedDict[tuple[int, int], Dict[str, mx.array]]" = OrderedDict() + self.prefetch_buffer_hits = 0 + self.prefetch_submitted = 0 + self.prefetch_dropped = 0 + # dual-source：decode 走 staging 双源命中计数（不写池的零拷贝命中） + self.staging_hits = 0 + # 可选 blob miss-loader（STREAM_BLOB_LOADER=1，由 model_builder 注入）： + # miss 时用每专家连续 blob 的并行 pread 取代 per-expert safetensors 的 mx.load， + # 复用常驻池 GPU-remap 快路径（命中零编排），小 EXPERT_SLOTS 即低内存。 + self._blob_loader = None + # 可选后台预取器（STREAM_BLOB_BG=1，由 model_builder 注入）：在独立 stream 上 + # 提前物化预测专家，promote_prefetched 把它们写进常驻池槽（主线程、便宜 scatter）。 + self._bg = None + + def cap_for(self, layer: int) -> int: + """该层常驻池容量(profile 指定则用之，否则全局 capacity)。""" + return self._resident.cap_for(layer) + + def resident_experts(self, layer: int) -> set[int]: + return self._resident.resident_experts(layer) + + def resident_lru_scores(self, layer: int) -> dict[int, float]: + return self._resident.resident_lru_scores(layer) + + @property + def hits(self) -> int: + return self._lru.hits + self.pinned_hits + self._resident.hits + + @property + def misses(self) -> int: + return self._lru.misses + self._resident.misses + + def resident_count(self) -> int: + pinned = sum(len(d) for d in self._pinned.values()) + return self._lru.resident_count() + pinned + + def pinned_count(self) -> int: + return sum(len(d) for d in self._pinned.values()) + + def path(self, layer: int, e: int) -> str: + import os + return os.path.join(self.root, f"layer{layer:02d}_expert{e:03d}.safetensors") + + def bundle_path(self, layer: int) -> str: + return os.path.join(self.bundle_dir, f"layer{layer:02d}.safetensors") + + def _load_layer_bundle(self, layer: int) -> Dict[str, mx.array]: + cached = self._bundle_cache.get(layer) + if cached is not None: + self._bundle_cache.move_to_end(layer) + return cached + bundle = mx.load(self.bundle_path(layer)) + self.bundle_loads += 1 + self._bundle_cache[layer] = bundle + self._bundle_cache.move_to_end(layer) + key_map: "Dict[int, list[tuple[str, str]]]" = {} + for full_key in bundle: + if not full_key.startswith("expert"): + continue + head, rest = full_key.split(".", 1) + e = int(head[len("expert"):]) + key_map.setdefault(e, []).append((rest, full_key)) + self._bundle_key_cache[layer] = key_map + while len(self._bundle_cache) > self.bundle_cache_size: + old_layer, _ = self._bundle_cache.popitem(last=False) + self._bundle_key_cache.pop(old_layer, None) + return bundle + + def _raw_load_one(self, layer: int, e: int) -> Dict[str, mx.array]: + if self._blob_loader is not None: + # blob 单专家：1 次连续 pread + 物化，结构与 mx.load 的 per-expert dict 一致。 + return self._blob_loader.load_experts(int(layer), [int(e)])[int(e)] + if self.use_bundles and os.path.exists(self.bundle_path(layer)): + bundle = self._load_layer_bundle(layer) + keys = self._bundle_key_cache.get(layer, {}).get(int(e), []) + return {short: bundle[full] for short, full in keys} + # 默认走 F_NOCACHE 读取(绕过 OS page cache):基准每次真实读盘、可复现。 + # mx.load 用 mmap 会经页缓存,重复跑被缓存命中污染→速度飘。EXPERT_NOCACHE=0 切回 mmap。 + if config.expert_nocache(): + w = load_safetensors_nocache(self.path(layer, e)) + else: + w = mx.load(self.path(layer, e)) + # 默认惰性:不在此 eval,让读盘并入 MLX 异步图、与计算重叠,token 末统一 eval。 + # 旧版每专家强制 mx.eval(w) 会每 token 触发 ~42 次同步、打碎流水线(实测 ~2.4× 慢)。 + # 数值完全等价(惰性求值仍精确)。EAGER_EXPERT_LOAD=1 可恢复旧的逐专家强制物化。 + if config.eager_expert_load(): + mx.eval(w) + return w + + def _load_one(self, layer: int, e: int) -> Dict[str, mx.array]: + return self._raw_load_one(layer, e) + + def _load_one_resident(self, layer: int, e: int) -> Dict[str, mx.array]: + """resident demand loader：优先消费 async prefetch buffer/inflight。""" + key = (int(layer), int(e)) + if self.async_prefetch: + cached = self._prefetch_buffer.pop(key, None) + if cached is not None: + self.prefetch_buffer_hits += 1 + self._resident.misses = max(0, self._resident.misses - 1) + return cached + return self._raw_load_one(layer, e) + + def _batch_load_resident(self, layer: int, ids: List[int]) -> Dict[int, Dict[str, mx.array]]: + """批量 resident demand loader：先消费 async prefetch buffer，剩余 miss 一次并行读。 + + 关键：blob 路径用一次 load_experts(layer, rest)（8-worker 并行 pread），取代逐专家 + 串行 pread（实测串行 6.4GB/s → 并行 8 路 22GB/s）。结果与逐专家加载等价。 + """ + ids = [int(e) for e in ids] + out: "Dict[int, Dict[str, mx.array]]" = {} + rest: List[int] = [] + for e in ids: + if self.async_prefetch: + cached = self._prefetch_buffer.pop((int(layer), e), None) + if cached is not None: + self.prefetch_buffer_hits += 1 + self._resident.misses = max(0, self._resident.misses - 1) + out[e] = cached + continue + rest.append(e) + if rest: + if self._blob_loader is not None: + out.update(self._blob_loader.load_experts(int(layer), rest)) + else: + for e in rest: + out[e] = self._raw_load_one(layer, e) + return out + + def _batch_load_stacked(self, layer: int, ids: List[int]) -> Dict[str, mx.array]: + """批量+预堆叠 demand loader：blob 路径一次 load_experts_stacked（并行 pread + 每段一次 + 物化），返回 {k:(N,*shape)} 按 ids 顺序。无 blob loader 时退化为逐专家物化后堆叠。 + + 仅在 async_prefetch 关闭时接入（不消费 prefetch buffer），保证不漏 buffer 命中。 + """ + ids = [int(e) for e in ids] + if self._blob_loader is not None: + return self._blob_loader.load_experts_stacked(int(layer), ids) + per = {e: self._raw_load_one(layer, e) for e in ids} + keys = list(per[ids[0]].keys()) + return {k: mx.stack([per[e][k] for e in ids], axis=0) for k in keys} + + def _batch_load_native(self, layer: int, ids: List[int]) -> "Dict[int, Dict[str, mx.array]]": + """批量 native demand loader:blob 路径用 C++ blob_load 把整批字节 pread 直进 MLX 数组 + (eval 时在 C++ 跑、绕 GIL、惰性),返回逐专家 {e:{proj.tensor:arr}}(供 _place_experts)。 + + 相比 numpy 路径(frombuffer+stack+mx.array 同步拷贝),把字节物化从 acquire 挪到 token 末 + eval、并绕开 GIL。无 blob loader 时退回逐专家 raw 物化(行为等价)。view_bf16=True 以 + 兼容 affine 量化(uint16 scales/biases→bf16);mxfp4 的 uint8 scales 忽略此参数。 + """ + ids = [int(e) for e in ids] + if self._blob_loader is not None: + return self._blob_loader.load_experts_native(int(layer), ids, view_bf16=True) + return {e: self._raw_load_one(layer, e) for e in ids} + + def note(self, layer: int, expert_ids: List[int]) -> None: + """记录一次激活（校准阶段调用），用于挑热专家。""" + c = self._counts.get(layer) + if c is None: + c = Counter() + self._counts[layer] = c + for e in expert_ids: + c[int(e)] += 1 + + def recorded_layers(self) -> List[int]: + return list(self._counts.keys()) + + def hot(self, layer: int, h: int) -> List[int]: + """返回该层激活最频繁的 h 个专家 id。""" + c = self._counts.get(layer) + if not c: + return [] + return [e for e, _ in c.most_common(h)] + + def pin(self, layer: int, expert_ids: List[int]) -> None: + """把这些专家加载并钉为常驻（不计入 LRU、不可驱逐）。""" + d = self._pinned.setdefault(layer, {}) + for e in expert_ids: + e = int(e) + if e not in d: + d[e] = self._load_one(layer, e) + # 连续 resident pool 是 decode/MTP 热路径；同步预取进去并标记不可驱逐， + # 否则旧 _pinned 只会影响 fetch/stack 路径，无法降低 acquire_gpu 的 miss。 + self._resident.pin_loaded(layer, {int(e): d[int(e)] for e in expert_ids}) + + def pin_batch(self, layer: int, expert_ids: List[int]) -> int: + """批量 pin（AUTOPIN 预热用）：语义同 pin，但加载收口为一次批量读，返回 pin 数。 + + 加载路径：有 blob_loader（生产唯一专家源）走 load_experts 一次 8-worker 并行 pread， + 取代 pin 的逐专家串行 per-expert safetensors（部署里 per-expert 文件已删，串行路径 + 会直接失败）；无 blob_loader 退回逐专家 _load_one（兼容旧 per-expert 部署）。 + dual(native demand)模式：真实区由 C++ g_real 权威，走 pin_loaded_dual 注册 real_pin + + C++ 直写池行，不留存数组副本（避免 pinned 专家双倍内存），fetch 路径不受影响。 + """ + layer = int(layer) + ids = list(dict.fromkeys(int(e) for e in expert_ids)) + if not ids: + return 0 + if getattr(self._resident, "_native_demand", False): + if self._blob_loader is not None: + loaded = self._blob_loader.load_experts(layer, ids) + else: + loaded = {e: self._load_one(layer, e) for e in ids} + return self._resident.pin_loaded_dual(layer, loaded) + d = self._pinned.setdefault(layer, {}) + todo = [e for e in ids if e not in d] + if todo: + if self._blob_loader is not None: + d.update(self._blob_loader.load_experts(layer, todo)) + else: + for e in todo: + d[e] = self._load_one(layer, e) + self._resident.pin_loaded(layer, {e: d[e] for e in ids}) + return len(ids) + + def prefetch(self, layer: int, expert_ids: List[int]) -> None: + """预取到连续 resident pool，不计入 demand miss/hit。""" + if not self.async_prefetch: + self._resident.prefetch(layer, expert_ids) + return + resident = self._resident.resident_experts(layer) + for e in dict.fromkeys(int(x) for x in expert_ids): + key = (int(layer), int(e)) + if e in resident: + self._resident.prefetch_hits += 1 + continue + if key in self._prefetch_buffer: + self._resident.prefetch_hits += 1 + continue + self._prefetch_buffer[key] = self._raw_load_one(int(layer), int(e)) + self._prefetch_buffer.move_to_end(key) + self.prefetch_submitted += 1 + self._resident.prefetch_loads += 1 + while len(self._prefetch_buffer) > self.prefetch_buffer_size: + self._prefetch_buffer.popitem(last=False) + self.prefetch_dropped += 1 + + def promote_prefetched(self, layer: int) -> int: + """把后台预取器中该层所有「已物化」专家写进常驻池槽（主线程、默认 stream）。 + + 同层(AHEAD=0)预取的就是本层将用到的专家，应真正放进池槽以转 miss 为 hit。 + 互保护：置入时把「本批就绪的全部专家」当作 current，使它们彼此不被驱逐， + 只挤掉池里陈旧的非本批专家；池满且无可驱逐者时跳过剩余，留给 demand 回退。 + 返回写入的专家数。 + """ + if self._bg is None: + return 0 + ready = self._bg.take_ready_layer(int(layer)) + self._bg.note_promote(int(layer), len(ready)) + if not ready: + return 0 + self._resident._ensure_layer(int(layer)) + slot_of = self._resident._slot_of[int(layer)] + protect = {int(e) for e in ready} + placed = 0 + for e, d in ready.items(): + e = int(e) + if e in slot_of: + continue + try: + self._resident._place_expert(int(layer), e, d, current=protect) + except ValueError: + # 池满且无非本批可驱逐 → 剩余专家留给 demand 路径 + break + placed += 1 + return placed + + def wait_prefetch(self) -> None: + """测试/诊断用：等待当前所有 async prefetch 完成。""" + return + + def release_fetch_cache(self) -> int: + """释放 fetch/stack 路径（prefill 用）的常驻缓存，返回释放的专家数。 + + decode 单 token 走 acquire_gpu/_resident 池，永不再碰 _lru/_stack_cache； + prefill 结束后这套缓存（每层 ≤cap 个专家 + 堆叠张量）是纯死重，约占 + cap×MoE层数 个专家的内存（实测 cap=16 时 ~9.2GB）。在 prefill→decode + 边界调用一次即可把这块预算还给系统/池。MTP 多 token 会按需重新填充。 + """ + freed = self._lru.resident_count() + self._lru._caches.clear() + self._stack_cache.clear() + clear_cache() + return freed + + def reset_stats(self) -> None: + self._lru.hits = 0 + self._lru.misses = 0 + self.pinned_hits = 0 + self._resident.hits = 0 + self._resident.misses = 0 + self._resident.gpu_fastpath = 0 + self._resident.gpu_fallback = 0 + self._resident.prefetch_hits = 0 + self._resident.prefetch_loads = 0 + self.prefetch_buffer_hits = 0 + self.prefetch_submitted = 0 + self.prefetch_dropped = 0 + self.staging_hits = 0 + + def acquire(self, layer: int, expert_ids: List[int]): + """连续常驻池取专家：返回 (pool_arrays, slots)。命中零拷贝，miss 单槽写。""" + if self.record: + self.note(layer, expert_ids) + return self._resident.acquire(layer, expert_ids) + + def acquire_gpu(self, layer: int, inds: mx.array, num_experts: int): + """GPU 侧 slot 重映射(decode 热路径):命中零 host 往返。record 模式才回 CPU 记账。""" + if self.record: + self.note(layer, [int(i) for i in inds.reshape(-1).tolist()]) + return self._resident.acquire_gpu(layer, inds, num_experts) + + def fetch(self, layer: int, expert_ids: List[int]) -> Dict[str, mx.array]: + if self.record: + self.note(layer, expert_ids) + key = tuple(int(e) for e in expert_ids) + cached = _stack_cache_get(self._stack_cache, self.stack_cache_size, layer, key) + if cached is not None: + pinned = self._pinned.get(layer) + for e in key: + if pinned is not None and e in pinned: + self.pinned_hits += 1 + else: + self._lru.get_or_load(layer, e, self._load_one) + return cached + pinned = self._pinned.get(layer) + picked = [] + for e in expert_ids: + e = int(e) + if pinned is not None and e in pinned: + self.pinned_hits += 1 + picked.append(pinned[e]) + else: + picked.append(self._lru.get_or_load(layer, e, self._load_one)) + stacked = _stack_picked(picked) + _stack_cache_put(self._stack_cache, self.stack_cache_size, layer, key, stacked) + return stacked + + def hit_rate(self) -> float: + tot = self.hits + self.misses + return self.hits / tot if tot else 0.0 diff --git a/mlx_streaming/core/cache/kv_quant_patch.py b/mlx_streaming/core/cache/kv_quant_patch.py new file mode 100644 index 0000000..e514149 --- /dev/null +++ b/mlx_streaming/core/cache/kv_quant_patch.py @@ -0,0 +1,116 @@ +"""把 Qwen3-Next 的 12 个全注意力层就地切换到 K4/V3 旋转量化 KV。 + +做法(隔离、低侵入): +- 子类 `_RotatedQuantAttn` 仅重写 `__call__`:RoPE 后对 q/k 旋转(R_k)、v 旋转(R_v), + 存非对称量化 cache,走 asym_quantized_sdpa,输出做 V 逆旋转复原。保参数树不变 + (只换 `__class__`、用 object.__setattr__ 挂常量,不进 nn.Module 的参数/子模块树)。 +- 覆盖 `model.make_cache`:全注意力层→AsymmetricQuantizedKVCache,线性层→ArraysCache(size=2)。 +""" +from __future__ import annotations + +import mlx.core as mx +from mlx_lm.models.cache import ArraysCache +from mlx_lm.models.qwen3_next import Qwen3NextAttention + +from mlx_streaming.core.cache.quant_kv import ( + AsymmetricQuantizedKVCache, + asym_quantized_sdpa, + build_block_so4, + rotate_last, +) + + +class _RotatedQuantAttn(Qwen3NextAttention): + """与原 Qwen3NextAttention.__call__ 数值等价(高 bit 时),叠加旋转 + 非对称量化。""" + + def __call__(self, x, mask=None, cache=None): + B, L, D = x.shape + + q_proj_output = self.q_proj(x) + queries, gate = mx.split( + q_proj_output.reshape(B, L, self.num_attention_heads, -1), 2, axis=-1) + gate = gate.reshape(B, L, -1) + + keys, values = self.k_proj(x), self.v_proj(x) + + queries = self.q_norm(queries).transpose(0, 2, 1, 3) + keys = self.k_norm( + keys.reshape(B, L, self.num_key_value_heads, -1)).transpose(0, 2, 1, 3) + values = values.reshape( + B, L, self.num_key_value_heads, -1).transpose(0, 2, 1, 3) + + if cache is not None: + queries = self.rope(queries, offset=cache.offset) + keys = self.rope(keys, offset=cache.offset) + else: + queries = self.rope(queries) + keys = self.rope(keys) + + # 旋转去相关(RoPE 之后):q、k 用 R_k(分数自动抵消),v 用 R_v(输出后逆旋转)。 + Rk = self._kvq_Rk + Rv = self._kvq_Rv + if Rk is not None: + queries = rotate_last(queries, Rk) + keys = rotate_last(keys, Rk) + values = rotate_last(values, Rv) + + if cache is not None: + keys, values = cache.update_and_fetch(keys, values) + output = asym_quantized_sdpa( + queries, keys, values, scale=self.scale, mask=mask, + group_size=self._kvq_gs, k_bits=self._kvq_kb, v_bits=self._kvq_vb) + else: + # 无 cache(罕见,如纯前向探针):走稠密 SDPA,V 仍在旋转空间,下方逆旋转复原。 + output = mx.fast.scaled_dot_product_attention( + queries, keys, values, scale=self.scale, mask=mask) + + if Rv is not None: + output = rotate_last(output, Rv.T) + + output = output.transpose(0, 2, 1, 3).reshape(B, L, -1) + return self.o_proj(output * mx.sigmoid(gate)) + + +def _inner(model): + """取到含 .layers 的内层模块(mlx_lm Model 包了一层 .model)。""" + return model.model if hasattr(model, "model") else model + + +def patch_kv_quant(model, *, group_size=64, k_bits=4, v_bits=3, rotate=True, seed=0): + """就地把所有全注意力层切到旋转 + 非对称量化 KV,并覆盖 make_cache。返回 model。""" + inner = _inner(model) + layers = inner.layers + + head_dim = None + for l in layers: + if not l.is_linear: + head_dim = l.self_attn.head_dim + break + if head_dim is None: + return model # 没有全注意力层,无需 patch + + Rk = build_block_so4(head_dim, seed=seed) if rotate else None + Rv = build_block_so4(head_dim, seed=seed + 1) if rotate else None + + for l in layers: + if l.is_linear: + continue + attn = l.self_attn + # 常量挂载用 object.__setattr__,避免进 nn.Module 参数/子模块树(否则破坏权重 save/树遍历)。 + object.__setattr__(attn, "_kvq_Rk", Rk) + object.__setattr__(attn, "_kvq_Rv", Rv) + object.__setattr__(attn, "_kvq_gs", group_size) + object.__setattr__(attn, "_kvq_kb", k_bits) + object.__setattr__(attn, "_kvq_vb", v_bits) + attn.__class__ = _RotatedQuantAttn + + def make_cache(): + return [ + AsymmetricQuantizedKVCache(group_size, k_bits, v_bits) + if not l.is_linear else ArraysCache(size=2) + for l in layers + ] + + # 覆盖实例方法(plain function,nn.Module.__setattr__ 不会把函数纳入参数树)。 + model.make_cache = make_cache + return model diff --git a/mlx_streaming/core/cache/quant_kv.py b/mlx_streaming/core/cache/quant_kv.py new file mode 100644 index 0000000..ea7a2e6 --- /dev/null +++ b/mlx_streaming/core/cache/quant_kv.py @@ -0,0 +1,223 @@ +"""IsoQuant 风格 K4/V3 非对称量化 KV cache(仅用于全注意力层)。 + +设计要点(详见 docs/superpowers/specs/2026-06-30-isoquant-kv-quant-design.md): +- SO(4) 块对角旋转去相关:head_dim=256 切成 64 个 4D 块,每块由两个单位四元数构造 + 一个 SO(4) 旋转(left·right_conj sandwich)。旋转正交,在注意力分数里自动抵消 + (q、k 同旋转 → q·kᵀ 不变);对 V 旋转后需在 SDPA 输出上做逆旋转复原。 +- 非对称仿射量化:K 用 k_bits(默认 4)、V 用 v_bits(默认 3),分别用 mx.quantize 打包, + attention 走 mx.quantized_matmul 快路径(每次调用可传不同 bits),无需自写 Metal。 + +打包宽度用 dim*bits//32(对 3-bit 必须如此:mlx_lm 自带 8*4//bits 公式对 3-bit 会算错)。 +""" +from __future__ import annotations + +import mlx.core as mx +import numpy as np +from mlx.utils import tree_map, tree_reduce +# 注意:make_mask 用的是 cache 模块里的 create_attention_mask(签名含 offset/return_array), +# 与 base 模块同名但不同签名的那个不可混用。 +from mlx_lm.models.cache import create_attention_mask + + +# ----------------------------- SO(4) 块旋转 ----------------------------- + +def _quat_left(q: np.ndarray) -> np.ndarray: + """单位四元数 q 的左乘矩阵 L(q):把 v 映射到 q*v(Hamilton 积)。""" + w, x, y, z = q + return np.array([ + [w, -x, -y, -z], + [x, w, -z, y], + [y, z, w, -x], + [z, -y, x, w], + ], dtype=np.float32) + + +def _quat_right_conj(q: np.ndarray) -> np.ndarray: + """单位四元数 q 的"右乘其共轭"矩阵 R(conj q):把 v 映射到 v*conj(q)。 + + L(qL)·R(conj qR) 即 SO(4) 的双边旋转 v -> qL * v * conj(qR),覆盖全部 4D 旋转。 + """ + w, x, y, z = q + return np.array([ + [ w, x, y, z], + [-x, w, z, -y], + [-y, -z, w, x], + [-z, y, -x, w], + ], dtype=np.float32) + + +def build_block_so4(head_dim: int, seed: int = 0, blocks_of: int = 4) -> mx.array: + """构造 head_dim×head_dim 的块对角正交矩阵,每 blocks_of(=4)维一个 SO(4) 旋转。 + + data-oblivious:仅由 seed 决定,固定可复现,不依赖数据分布。 + """ + assert head_dim % blocks_of == 0, "head_dim 必须被 blocks_of 整除" + rng = np.random.default_rng(seed) + M = np.zeros((head_dim, head_dim), dtype=np.float32) + for b in range(head_dim // blocks_of): + qL = rng.standard_normal(4).astype(np.float32) + qL /= np.linalg.norm(qL) + qR = rng.standard_normal(4).astype(np.float32) + qR /= np.linalg.norm(qR) + blk = _quat_left(qL) @ _quat_right_conj(qR) # 4×4 ∈ SO(4) + i = b * blocks_of + M[i:i + blocks_of, i:i + blocks_of] = blk + return mx.array(M) + + +def rotate_last(x: mx.array, R: mx.array) -> mx.array: + """对末维做正交旋转:x[..., D] @ R[D, D](按 x 的 dtype 计算)。""" + return x @ R.astype(x.dtype) + + +# --------------------- 非对称量化 KV cache(K4/V3)--------------------- + +def _packed_words(dim: int, bits: int) -> int: + """量化后每行占的 uint32 个数:dim*bits//32(对 2/3/4/5/6/8 bit 均正确)。""" + return dim * bits // 32 + + +class AsymmetricQuantizedKVCache: + """K、V 各用独立位宽的量化 KV cache(仿 mlx_lm.QuantizedKVCache,但 K/V 不同 bits)。 + + keys/values 各存为 (wq, scales, biases) 三元组,与 mx.quantized_matmul 接口对齐。 + """ + + step = 256 + + def __init__(self, group_size: int = 64, k_bits: int = 4, v_bits: int = 3): + self.keys = None + self.values = None + self.offset = 0 + self.group_size = group_size + self.k_bits = k_bits + self.v_bits = v_bits + + def _init_quant(self, dim, bits, B, H, steps, dt): + packed = _packed_words(dim, bits) + ngroups = dim // self.group_size + return ( + mx.zeros((B, H, steps, packed), dtype=mx.uint32), + mx.zeros((B, H, steps, ngroups), dtype=dt), + mx.zeros((B, H, steps, ngroups), dtype=dt), + ) + + def update_and_fetch(self, keys, values): + B, H, n, kD = keys.shape + vD = values.shape[-1] + prev = self.offset + + if self.keys is None or (prev + n) > self.keys[0].shape[-2]: + new_steps = (self.step + n - 1) // self.step * self.step + if self.keys is not None: + if prev % self.step != 0: + self.keys = tree_map(lambda x: x[..., :prev, :], self.keys) + self.values = tree_map(lambda x: x[..., :prev, :], self.values) + + def _expand(x): + z = mx.zeros((B, H, new_steps, x.shape[-1]), dtype=x.dtype) + return mx.concatenate([x, z], axis=-2) + + self.keys = tree_map(_expand, self.keys) + self.values = tree_map(_expand, self.values) + else: + self.keys = self._init_quant(kD, self.k_bits, B, H, new_steps, keys.dtype) + self.values = self._init_quant(vD, self.v_bits, B, H, new_steps, values.dtype) + + self.offset += n + kq = mx.quantize(keys, group_size=self.group_size, bits=self.k_bits) + vq = mx.quantize(values, group_size=self.group_size, bits=self.v_bits) + for i in range(3): + self.keys[i][..., prev:self.offset, :] = kq[i] + self.values[i][..., prev:self.offset, :] = vq[i] + + return ( + tree_map(lambda x: x[..., :self.offset, :], self.keys), + tree_map(lambda x: x[..., :self.offset, :], self.values), + ) + + def make_mask(self, *args, **kwargs): + return create_attention_mask(*args, offset=self.offset, **kwargs) + + def is_trimmable(self): + return True + + def trim(self, n): + n = min(self.offset, n) + self.offset -= n + return n + + def empty(self): + return self.keys is None + + @property + def nbytes(self): + if self.keys is None: + return 0 + return tree_reduce(lambda a, x: a + x.nbytes, (self.keys, self.values), 0) + + # ---- 快照接口(与 KVCache 同口径,供 mtp.kv_cache._snapshot/_restore 与前缀快照使用)---- + @property + def state(self): + # buffer 未满时切到 offset(MLX 切片产生新数组,快照不会被后续原地写污染) + if self.offset == self.keys[0].shape[-2]: + return self.keys, self.values + return ( + tree_map(lambda x: x[..., : self.offset, :], self.keys), + tree_map(lambda x: x[..., : self.offset, :], self.values), + ) + + @state.setter + def state(self, v): + self.keys, self.values = v + self.offset = self.keys[0].shape[-2] + + @property + def meta_state(self): + return str(self.group_size), str(self.k_bits), str(self.v_bits) + + @meta_state.setter + def meta_state(self, v): + self.group_size, self.k_bits, self.v_bits = map(int, v) + + +# ----------------- 非对称量化 SDPA(K 用 k_bits、V 用 v_bits)----------------- + +def asym_quantized_sdpa(queries, q_keys, q_values, scale, mask, + group_size, k_bits, v_bits): + """基于 mlx_lm.quantized_scaled_dot_product_attention,K/V 各自传 bits。 + + queries: [B, n_q_heads, L, D];q_keys/q_values: 量化三元组(n_kv_heads)。 + 返回 [B, n_q_heads, L, D](仍在旋转后的 V 空间,调用方负责逆旋转)。 + """ + B, n_q_heads, L, D = queries.shape + n_kv_heads = q_keys[0].shape[-3] + n_repeats = n_q_heads // n_kv_heads + + queries = queries * scale + + if n_repeats > 1: + queries = mx.reshape(queries, (B, n_kv_heads, n_repeats, L, D)) + q_keys = tree_map(lambda x: mx.expand_dims(x, axis=-3), q_keys) + q_values = tree_map(lambda x: mx.expand_dims(x, axis=-3), q_values) + + scores = mx.quantized_matmul( + queries, *q_keys, transpose=True, group_size=group_size, bits=k_bits) + if mask is not None: + if isinstance(mask, str): + qL, kL = scores.shape[-2:] + q_indices = mx.arange(kL - qL, kL) + k_indices = mx.arange(kL) + mask = q_indices[:, None] >= k_indices[None] + if mask.dtype == mx.bool_: + scores = mx.where(mask, scores, mx.finfo(scores.dtype).min) + else: + scores = scores + mask + scores = mx.softmax(scores, axis=-1, precise=True) + out = mx.quantized_matmul( + scores, *q_values, transpose=False, group_size=group_size, bits=v_bits) + + if n_repeats > 1: + out = mx.reshape(out, (B, n_q_heads, L, D)) + + return out diff --git a/mlx_streaming/core/cache/resident_pool.py b/mlx_streaming/core/cache/resident_pool.py new file mode 100644 index 0000000..75b25a2 --- /dev/null +++ b/mlx_streaming/core/cache/resident_pool.py @@ -0,0 +1,635 @@ +"""常驻专家池：每层一块连续 GPU 张量 + slot LRU，支持按需增长、pin、GPU 侧重映射。 + +设计要点： +- 命中只返回槽位、不写池；miss 只把单个专家原地写进它的槽位（`_write_slot`），避免拷贝整池。 +- 池按需增长（grow-on-demand）：起步 `_POOL_INIT_SLOTS` 行，工作集扩大时 ~1.5× 增至天花板 + `cap_for(layer)` 才开始 LRU 淘汰。默认无 profile 即自动右尺寸，容量内永不超预算。 +- `acquire_gpu`：decode 热路径用 GPU 查找表做纯 GPU slot 重映射，命中层零 host 往返， + 仅一次 miss 标志同步；真 miss 才回退 host 读盘路径。 +""" +import os +from collections import OrderedDict, Counter +from typing import Dict, List + +import mlx.core as mx + +# 诊断门控：staging/acquire_gpu 路径消费侧字节级真值校验（混合 ahead 损坏取证用，默认关）。 +# 在 acquire_gpu 全命中快路径返回前，对本层真实路由命中的每个专家，把池槽字节与磁盘真值 +# 逐 key 比对；不一致即「池槽装错字节」铁证。受 STG_VERIFY=1 控制，对主路径零影响（默认 off）。 +_STG_VERIFY = os.environ.get("STG_VERIFY") == "1" +_stg_verify_state = {"ok": 0, "bad": 0, "printed": 0, "calls": 0, "first_bad_call": None} + +from mlx_streaming import config +from mlx_streaming.core.cache import autopin + +# Route 3 Phase 1 底座：spec/dual 模式下池 buffer 改由 C++ 拥有(mx::allocator + no-op deleter)， +# 地址进程内恒定、永不被 MLX donation/迁移，供侧区/demand 后台 pread 安全直写（替代消费侧 MLX scatter）。 +# POOL_OWNED=0 可强制回退 mx.zeros(仅供 A/B 对照)。 +_POOL_OWNED = os.environ.get("POOL_OWNED", "1") == "1" +# mx.Dtype -> C++ pool_owned_zeros 接受的 dtype 名。 +_DTYPE_NAME = { + mx.uint32: "uint32", mx.uint16: "uint16", mx.uint8: "uint8", + mx.int32: "int32", mx.int16: "int16", + mx.bfloat16: "bfloat16", mx.float16: "float16", mx.float32: "float32", +} + + +def _owned_pool(sample: "Dict[str, mx.array]", n: int) -> "Dict[str, mx.array]": + """用 C++-owned buffer 建 (n,*shape) 的 per-key 池数组（地址恒定，供 C++ 直写）。""" + import mlx_streaming.native_moe_ext as _N + out = {} + for k, v in sample.items(): + name = _DTYPE_NAME.get(v.dtype) + if name is None: + raise RuntimeError(f"pool_owned_zeros 不支持 dtype {v.dtype} (key={k})") + out[k] = _N.pool_owned_zeros([int(n)] + [int(d) for d in v.shape], name) + return out + + +# 池按需增长(grow-on-demand)的初始物理行数。 +# 起步小,工作集扩大时按 ~1.5× 增长,封顶 cap_for(默认=全局 capacity)。 +# 好处:无需 profile 也能自动右尺寸(内存≈实际工作集),且对任何 prompt 自适应、容量内永不超预算。 +_POOL_INIT_SLOTS = 16 + + +class ResidentExpertPool: + """每层一个连续常驻池：(capacity,*shape) 张量 + slot LRU。 + + 命中只返回槽位、不写池；miss 只把单个专家原地写进它的槽位（_write_slot）。 + loader(layer, e) -> Dict[str, mx.array]，单个专家的参数（未堆叠）。 + """ + + def __init__(self, capacity: int, loader, layer_caps: "Dict[int, int] | None" = None, + spec_slots: int = 0, batch_loader=None, stacked_batch_loader=None, + spec_gens: int = 1): + self.capacity = capacity + self.loader = loader + # 可选批量加载器 batch_loader(layer, [ids]) -> {e: expert_dict}：acquire 用它把本层所有 + # miss 一次并行读（8-worker pread），取代逐专家串行 loader。None 时退回串行 loader。 + self.batch_loader = batch_loader + # 可选「批量+预堆叠」加载器 stacked_batch_loader(layer, [ids]) -> {k:(N,*shape)}： + # 在 batch_loader 基础上再把 6N 次碎 mx.array 物化折成每段一次，写池时直接整批 scatter， + # 既省 frombuffer/mx.array 构造又省 _write_slots_batch 里的 mx.stack。优先级最高。 + self.stacked_batch_loader = stacked_batch_loader + self.spec_slots = int(spec_slots) # >0：侧区模式（预分配 cap+spec、禁 grow） + self.spec_gens = max(1, int(spec_gens)) # 侧区代数：双缓冲=2，物理行=cap+spec_gens*spec_slots + # cap_for(layer) 是该层物理行数的「天花板」：profile 指定则用之，否则=全局 capacity。 + # 池按需增长(grow-on-demand)：起步小、随工作集扩大增至天花板才开始 LRU 淘汰。 + # 因此默认(无 profile)即自动右尺寸——内存≈实际工作集,对任意 prompt 自适应, + # 且天花板内永不超预算(超预算只是更慢,不崩)。增长偶发(预热期),稳态零拷贝。 + self.layer_caps: "Dict[int, int]" = dict(layer_caps or {}) + self._pools: "Dict[int, Dict[str, mx.array]]" = {} + self._slot_of: "Dict[int, OrderedDict[int, int]]" = {} + self._free: "Dict[int, list]" = {} + # 每层当前物理已分配行数(按需增长,≤ cap_for)。grow-on-demand 的核心状态。 + self._alloc: "Dict[int, int]" = {} + # 每层 GPU 查找表(全局专家 id → slot,-1=不在池),供 acquire_gpu 在命中层 + # 做纯 GPU 重映射(消除每层 .tolist 同步)。与 _slot_of 同步维护(仅在表已建时)。 + self._slot_table: "Dict[int, mx.array]" = {} + # 每层 pinned 专家集合：预取进 resident pool 后不参与 LRU 驱逐。 + # 用于 K>2 / 小槽位时保住热专家工作集，降低真实 miss。 + self._pinned: "Dict[int, set[int]]" = {} + # 可选替换策略：默认 lfu（短窗口频率 + LRU tie-break，实测比 lru 命中更高）；EVICT_POLICY=lru 回退纯 LRU。 + self.eviction_policy = config.evict_policy().lower() + self.lfu_decay_interval = config.lfu_decay_interval() + self._freq: "Dict[int, Counter[int]]" = {} + self._access_count: "Dict[int, int]" = {} + self.hits = 0 + self.misses = 0 + self.prefetch_hits = 0 + self.prefetch_loads = 0 + # GPU remap 路径取证:命中层走纯 GPU 快路径次数 vs 有 miss 回退 host 次数 + self.gpu_fastpath = 0 + self.gpu_fallback = 0 + # 累计「分不到槽被迫落 0 号槽」的专家数(来自 demand_dual)。任何 >0 都意味着有前向 + # 拿错专家权重算过 → 输出不可信,必须靠调小 PREFILL_CHUNK / 调大槽位消除。 + self.unplaced = 0 + # dual 路径真实区槽状态由 C++ demand_dual 唯一权威(无 opt-out)。spec 模式 + native 已编译即启用; + # 非 spec(spec_slots==0)或 native 缺失时保持 Python 权威路径(仅 prefill/host/非双源用)。 + # 启用后 _slot_of/_free/_freq 在 dual 路径不再维护,resident_experts/_count 改查 C++ g_real(预取过滤要用)。 + self._native_demand = False + if int(spec_slots) > 0: + try: + import mlx_streaming.native_moe_ext as _N + self._native_demand = hasattr(_N, "demand_dual") + except Exception: + self._native_demand = False + if self._native_demand: + # 复刻基线 8-worker 并行读：demand miss 的 pread 派给 BgReader 并行执行（高优队列）。 + import mlx_streaming.native_moe_ext as _N + try: + _N.bg_reader_start(int(os.environ.get("DEMAND_WORKERS", "8")), 0) + except Exception: + pass + if self._native_demand and os.environ.get("DEMAND_TIMING") == "1": + import atexit + import mlx_streaming.native_moe_ext as _N + _N.demand_timing_enable(True) + atexit.register(lambda: print( + "[DEMAND_TIMING ms] inds/pool/side_snap/real_lock/core/build =", + [round(x / 1e3, 1) for x in _N.demand_timings()], flush=True)) + + def cap_for(self, layer: int) -> int: + """该层池容量：profile 指定则用之(上限 capacity)，否则用全局 capacity。""" + return min(self.layer_caps.get(layer, self.capacity), self.capacity) + + def _ensure_layer(self, layer: int): + if layer not in self._slot_of: + self._slot_of[layer] = OrderedDict() + # 空闲槽列表懒填:首次 miss 分配池时再灌入,之后按需增长时追加。 + # 注意:始终原地 mutate 这个 list,绝不重新绑定,确保 acquire 里持有的本地引用同步可见。 + self._free[layer] = [] + self._pinned[layer] = set() + self._freq[layer] = Counter() + self._access_count[layer] = 0 + + def _alloc_pool(self, layer: int, sample: Dict[str, mx.array]): + """首次为某层分配池张量,起步 _POOL_INIT_SLOTS 行(不超过该层天花板)。""" + if self.spec_slots > 0: + n = self.cap_for(layer) + self.spec_gens * self.spec_slots # 预分配满（含 spec_gens 代侧区） + # spec/dual 模式：池由 C++ 拥有，侧区异步直写落此 buffer（Route 3 底座）。 + if _POOL_OWNED: + self._pools[layer] = _owned_pool(sample, n) + else: + self._pools[layer] = { + k: mx.zeros((n,) + v.shape, dtype=v.dtype) for k, v in sample.items()} + else: + n = min(_POOL_INIT_SLOTS, self.cap_for(layer)) + self._pools[layer] = { + k: mx.zeros((n,) + v.shape, dtype=v.dtype) + for k, v in sample.items() + } + self._alloc[layer] = n + real = self.cap_for(layer) if self.spec_slots > 0 else n + self._free[layer].extend(range(real)) # 原地 mutate,不重新绑定；侧区行不进 free，永不被 LRU 分配/驱逐 + + def _grow_pool(self, layer: int, new_n: int): + """把某层池物理行数扩到 new_n(封顶 cap_for),拼接保留已驻行的数据与 slot 索引。""" + if self.spec_slots > 0: + return # 侧区模式：预分配满，永不 grow（保 C++ 写指针稳定） + old_n = self._alloc[layer] + new_n = min(new_n, self.cap_for(layer)) + if new_n <= old_n: + return + old = self._pools[layer] + self._pools[layer] = { + k: mx.concatenate( + [v, mx.zeros((new_n - old_n,) + v.shape[1:], dtype=v.dtype)], + axis=0) + for k, v in old.items() + } + self._free[layer].extend(range(old_n, new_n)) # 新行追加为空闲,原地 mutate + self._alloc[layer] = new_n + + def preallocate(self, layer: int, sample: "Dict[str, mx.array]", cap: int): + """满 cap 预分配 typed 池并 eval 固定 data 指针；幂等。 + + 供"C++ 直写槽"机制使用：调用后该层不再 grow/不再走 _place_expert(MLX scatter)， + 槽位由调用方直接 mutate _slot_of + _set_table 管理、字节由 C++ pread 写入。 + """ + self._ensure_layer(layer) + if layer in self._pools: + return + if _POOL_OWNED: + self._pools[layer] = _owned_pool(sample, cap) # C++ 拥有，地址恒定供直写 + else: + self._pools[layer] = { + k: mx.zeros((cap,) + v.shape, dtype=v.dtype) for k, v in sample.items() + } + mx.eval(list(self._pools[layer].values())) # 固定 data 指针 + self._alloc[layer] = cap + # 登记所有物理行为空闲,供共用分配器(_alloc_slot)按 free 优先分配； + # 已被 slot_of 占用的行不在此列(暖池复用时 slot_of 已满 → free 为空 → 走驱逐)。 + occupied = set(self._slot_of[layer].values()) + self._free[layer].extend(r for r in range(cap) if r not in occupied) + + def allocated_slots(self, layer: int) -> int: + """该层池张量当前物理行数(按需增长,≤ cap_for),用于内存核算/测试。""" + return self._alloc.get(layer, 0) + + def _pool_owned(self, layer: int) -> bool: + """该层池是否为 C++-owned buffer（spec/dual 模式 + POOL_OWNED）。 + owned 池禁止任何 MLX scatter（会重分配 buffer、孤立侧区直写），落池一律走 C++ 直写。""" + return _POOL_OWNED and self.spec_slots > 0 + + def _write_slot(self, layer: int, slot: int, expert: Dict[str, mx.array]): + if self._pool_owned(layer): + self._write_slots_batch(layer, [slot], [expert]) # 单槽也走 C++ 直写，避免 scatter + return + pool = self._pools[layer] + for k, v in expert.items(): + pool[k][slot] = v # de-risk 选定：原地写，单槽、不拷贝整池 + + def _write_slots_batch(self, layer: int, slots: List[int], + experts: "List[Dict[str, mx.array]]") -> None: + """把多个专家一次性写进各自槽位。 + + owned 池：C++ memcpy 直写各行（无 MLX scatter，保 buffer 地址恒定 → 侧区直写不被孤立）。 + 非 owned：每个 key 一次 stack + fancy-index scatter（原碎 kernel 最少化路径）。 + """ + if not slots: + return + pool = self._pools[layer] + if self._pool_owned(layer): + import mlx_streaming.native_moe_ext as _N + keys = list(pool.keys()) + pool_list = [pool[k] for k in keys] + srcs_flat = [e[k] for k in keys for e in experts] # key-major：与 C++ pool_write_rows 约定一致 + mx.eval(srcs_flat) # 物化源，固定 data 指针 + _N.pool_write_rows(pool_list, srcs_flat, [int(s) for s in slots]) + return + idx = mx.array(slots, dtype=mx.int32) + for k in pool: + pool[k][idx] = mx.stack([e[k] for e in experts], axis=0) + + def resident_count(self, layer: int) -> int: + if self._native_demand: # 方案B：C++ g_real 为真实区权威 + import mlx_streaming.native_moe_ext as N + return int(N.real_region_count(int(layer))) + return len(self._slot_of.get(layer, ())) + + def resident_experts(self, layer: int) -> set[int]: + """返回该层 resident pool 里当前已有的专家 id 集合（trace/probe + 侧区预取过滤用）。""" + if self._native_demand: # 方案B：查 C++ g_real，保预取过滤一致 + import mlx_streaming.native_moe_ext as N + flat = N.real_region_contents(int(layer)) + return {int(flat[i]) for i in range(0, len(flat), 2)} + return set(self._slot_of.get(layer, {}).keys()) + + def _bootstrap_dual_pool(self, layer: int) -> None: + """方案B：首次为该层预分配满(cap+spec_gens*spec)池、eval 固定 data 指针 + C++ real_init。幂等。 + + 池结构(per-key typed 数组)由 Python 从一个样本专家的 shape 建；字节后续由 C++ demand pread 写入。 + Python 侧 _slot_of/_free 在方案B dual 路径不再使用（真实区状态归 C++ g_real）。 + """ + self._ensure_layer(layer) + if layer not in self._pools: + sample = self.loader(layer, 0) # 仅取 shape/dtype，不写入池 + cap = self.cap_for(layer) + n = cap + self.spec_gens * self.spec_slots + if _POOL_OWNED: + self._pools[layer] = _owned_pool(sample, n) # C++ 拥有，地址恒定供直写 + else: + self._pools[layer] = { + k: mx.zeros((n,) + v.shape, dtype=v.dtype) for k, v in sample.items() + } + mx.eval(list(self._pools[layer].values())) # 固定 data 指针，供 C++ memcpy + self._alloc[layer] = n + import mlx_streaming.native_moe_ext as N + N.real_init(int(layer), int(cap)) + + def resident_lru_scores(self, layer: int) -> dict[int, float]: + """返回 resident 专家的 LRU 新近度分数：0=最久未用，1=最近使用。""" + keys = list(self._slot_of.get(layer, {}).keys()) + if not keys: + return {} + denom = max(1, len(keys) - 1) + return {int(e): i / denom for i, e in enumerate(keys)} + + def _note_access(self, layer: int, expert_ids: List[int]) -> None: + if self.eviction_policy != "lfu": + return + self._ensure_layer(layer) + freq = self._freq[layer] + for e in expert_ids: + freq[int(e)] += 1 + self._access_count[layer] += len(expert_ids) + if self.lfu_decay_interval > 0 and self._access_count[layer] >= self.lfu_decay_interval: + for e in list(freq): + freq[e] //= 2 + if freq[e] <= 0: + del freq[e] + self._access_count[layer] = 0 + + def _choose_victim(self, layer: int, current: set[int]) -> int: + slot_of = self._slot_of[layer] + pinned = self._pinned.get(layer, set()) + # 绝不驱逐当前请求(current)专家：否则 acquire 末尾按 slot_of 取槽会 KeyError。 + # 只在「非 current 且非 pinned」中选受害者；选不出来说明本次唯一专家+不可驱逐者已超容量， + # 由调用方(acquire 的 len(uniq)>cap 守卫)负责拦截，这里直接报清晰错误。 + candidates = [e for e in slot_of if e not in pinned and e not in current] + if not candidates: + raise ValueError( + f"layer {layer} resident pool has no evictable non-current slot " + f"(capacity={self.cap_for(layer)}, pinned={len(pinned)}, current={len(current)})") + if self.eviction_policy != "lfu": + return candidates[0] + freq = self._freq.get(layer, Counter()) + return min(enumerate(candidates), key=lambda x: (freq.get(x[1], 0), x[0]))[1] + + def _alloc_slot(self, layer: int, e: int, current: "set[int]") -> "tuple[int, bool]": + """为 e 分配/复用槽（free 优先，否则驱逐非 pinned/非 current 的 LRU 最旧）， + 更新 slot_of + table，但不写数据。返回 (slot, is_new)。 + + 供 C++ 直写预取(prefetch_cpp)与 demand 写入(_place_expert)**共用**：保证两条路径 + 从同一套 _free/_slot_of/_slot_table 分配，绝不抢占同一物理槽。 + """ + slot_of, free = self._slot_of[layer], self._free[layer] + if e in slot_of: + slot_of.move_to_end(e) + return slot_of[e], False + cap = self.cap_for(layer) + if not free and self._alloc.get(layer, 0) < cap: + cur = self._alloc[layer] + self._grow_pool(layer, cur + max(1, cur // 2)) + if free: + slot = free.pop(0) + else: + evicted_e = self._choose_victim(layer, current) + slot = slot_of.pop(evicted_e) + self._clear_table(layer, evicted_e) + slot_of[e] = slot + slot_of.move_to_end(e) + self._set_table(layer, e, slot) + return slot, True + + def _place_expert(self, layer: int, e: int, expert: Dict[str, mx.array], + current: "set[int] | None" = None) -> int: + """把专家写进 resident pool，必要时增长或驱逐非 pinned LRU，返回 slot。""" + current = current or {e} + if layer not in self._pools: + self._alloc_pool(layer, expert) + slot, _ = self._alloc_slot(layer, e, current) + self._write_slot(layer, slot, expert) + return slot + + def _place_experts(self, layer: int, ids: List[int], + experts: "Dict[int, Dict[str, mx.array]]", + current: "set[int]") -> List[int]: + """批量把一组 miss 专家写进 resident pool：共用槽分配器 + 单次批量 scatter。 + + 与逐个 _place_expert 语义等价（同样的 grow/驱逐/槽分配顺序）， + 只把数据写入合并成 _write_slots_batch，去掉每 token 上百次碎 scatter。 + """ + if not ids: + return [] + if layer not in self._pools: + self._alloc_pool(layer, experts[ids[0]]) + slots = [self._alloc_slot(layer, e, current)[0] for e in ids] + self._write_slots_batch(layer, slots, [experts[e] for e in ids]) + return slots + + def _place_experts_stacked(self, layer: int, ids: List[int], + stacked: "Dict[str, mx.array]", + current: "set[int]") -> List[int]: + """批量写一组 miss：stacked 为按 ids 顺序预堆叠的 {k:(N,*shape)}， + 每 key 直接一次 fancy-index scatter（连 _write_slots_batch 的 mx.stack 都省了）。 + """ + if not ids: + return [] + if layer not in self._pools: + self._alloc_pool(layer, {k: v[0] for k, v in stacked.items()}) + slots = [self._alloc_slot(layer, e, current)[0] for e in ids] + pool = self._pools[layer] + if self._pool_owned(layer): + import mlx_streaming.native_moe_ext as _N + keys = list(pool.keys()) + pool_list = [pool[k] for k in keys] + stacked_list = [stacked[k] for k in keys] + mx.eval(stacked_list) # 物化预堆叠源 + _N.pool_write_stacked(pool_list, stacked_list, [int(s) for s in slots]) + return slots + idx = mx.array(slots, dtype=mx.int32) + for k in pool: + pool[k][idx] = stacked[k] + return slots + + def prefetch_cpp(self, layer: int, expert_ids, submit_fn) -> list: + """用 C++ 直写把预测专家投机预取进池（与 demand fallback 共用槽分配器）。 + + submit_fn(slot, expert)：提交一次异步 C++ 后台读到该物理槽（调用方负责后续 wait）。 + 已驻 → 触摸 LRU 跳过；未驻 → 分配槽并 submit。返回新提交的 expert 列表。 + 投机性质：不计 hit/miss，分得的槽可被后续 LRU 驱逐；无可驱逐槽时跳过该专家 + （交给 demand fallback），绝不抛错打断热路径。 + """ + self._ensure_layer(layer) + if layer not in self._pools: + return [] # 池未建（首 token 预热）→ 全交 fallback + current = {int(e) for e in expert_ids} + slot_of = self._slot_of[layer] + submitted = [] + for e in expert_ids: + e = int(e) + if e in slot_of: + slot_of.move_to_end(e) + continue + try: + slot, _ = self._alloc_slot(layer, e, current) + except ValueError: + continue # 无可驱逐槽 → 跳过，留给 demand fallback + submit_fn(slot, e) + submitted.append(e) + return submitted + + def pin(self, layer: int, expert_ids: List[int]) -> None: + """预取并钉住专家到 resident pool；pinned 专家不被 LRU 驱逐。 + + pin 是显式预取，不计入 hit/miss 统计；后续 acquire/acquire_gpu 命中才计 hit。 + """ + # dual 模式(demand_dual 唯一权威)已弃用 PIN_HOT:真实区归 C++ g_real,Python pinned 不再生效。 + # 明确报错而非静默无效——请设 PIN_HOT=0。 + if self._native_demand and list(expert_ids): + raise RuntimeError( + "PIN_HOT 在 demand_dual(dual 模式)下已弃用:真实区由 C++ g_real 权威、不支持 pin。" + "请设 PIN_HOT=0。") + loaded = {int(e): self.loader(layer, int(e)) for e in expert_ids} + self.pin_loaded(layer, loaded) + + def pin_loaded(self, layer: int, experts: "Dict[int, Dict[str, mx.array]]") -> None: + """把已加载专家钉进 resident pool，避免 FileExpertStore.pin 重复读/持有两份。""" + uniq = list(dict.fromkeys(int(e) for e in experts)) + cap = self.cap_for(layer) + self._ensure_layer(layer) + pinned = self._pinned[layer] + if len(pinned | set(uniq)) > cap: + raise ValueError( + f"pin 后 pinned 数量 {len(pinned | set(uniq))} > 该层池容量 {cap}") + slot_of = self._slot_of[layer] + for e in uniq: + if e in slot_of: + pinned.add(e) + slot_of.move_to_end(e) + continue + self._place_expert(layer, e, experts[e], current={e}) + pinned.add(e) + + def pin_loaded_dual(self, layer: int, experts: "Dict[int, Dict[str, mx.array]]") -> int: + """dual(native demand)模式的 pin：C++ g_real 是真实区唯一权威（Python _slot_of 不生效）， + 必须 real_pin 注册（pinned 免驱逐 + 从 free 头取真实区槽）再用 pool_write_rows 把字节 + C++ 直写池行（owned 池禁止 MLX scatter）。返回成功 pin 的专家数。 + + 仅在 _native_demand 下由 FileExpertStore.pin_batch 调用；experts 为已加载的 + {expert: {key: arr}}（key 集/顺序与池一致——同源自 blob 段表）。 + """ + import mlx_streaming.native_moe_ext as _N + ids = list(dict.fromkeys(int(e) for e in experts)) + if not ids: + return 0 + self._bootstrap_dual_pool(layer) # 建池 + real_init（幂等） + slots = _N.real_pin(int(layer), ids) # 登记 pinned + 分配真实区槽（-1=分不到） + ok = [(e, s) for e, s in zip(ids, slots) if s >= 0] + if not ok: + return 0 + keys = list(self._pools[layer].keys()) + pool_list = [self._pools[layer][k] for k in keys] + srcs_flat = [experts[e][k] for k in keys for e, _ in ok] # key-major，同 _write_slots_batch + mx.eval(srcs_flat) + _N.pool_write_rows(pool_list, srcs_flat, [s for _, s in ok]) + return len(ok) + + def prefetch(self, layer: int, expert_ids: List[int]) -> None: + """把专家预取进 resident pool，但不计入 hit/miss，且可被后续 LRU 驱逐。""" + uniq = list(dict.fromkeys(int(e) for e in expert_ids)) + cap = self.cap_for(layer) + if len(uniq) > cap: + uniq = uniq[:cap] + self._ensure_layer(layer) + slot_of = self._slot_of[layer] + current = set(uniq) + for e in uniq: + if e in slot_of: + slot_of.move_to_end(e) + self.prefetch_hits += 1 + continue + expert = self.loader(layer, e) + self._place_expert(layer, e, expert, current=current) + self.prefetch_loads += 1 + + def acquire(self, layer: int, expert_ids: List[int], protect: "set[int] | None" = None): + """protect：额外的「不可驱逐」专家集（除本批 miss 外）。dual 路径传入本前向全部 inds， + 确保落 miss 腾槽时不驱逐本前向仍需的命中专家（否则其 slot 被复用 → 消费侧 gather 错字节）。 + 仅影响驱逐候选，不改 hit/miss 记账口径。""" + # 唯一专家集合(保序去重)：池只需同时容纳本次请求的唯一专家 + uniq = list(dict.fromkeys(int(e) for e in expert_ids)) + cap = self.cap_for(layer) + if len(uniq) > cap: + raise ValueError( + f"本次请求 {len(uniq)} 个唯一专家 > 该层池容量 {cap}") + self._ensure_layer(layer) + self._note_access(layer, uniq) + uniq_set = set(uniq) + # 驱逐保护集 = 本批 miss ∪ 调用方额外指定(如本前向全部 inds 命中专家)。 + current = uniq_set if protect is None else (uniq_set | protect) + slot_of, free = self._slot_of[layer], self._free[layer] + # 先处理命中（触摸 LRU），再把所有 miss 收成一批 + misses = [] + for e in uniq: + if e in slot_of: + self.hits += 1 + slot_of.move_to_end(e) # 触摸为最近使用，避免本次内被驱逐 + else: + misses.append(e) + if misses: + self.misses += len(misses) + if self.stacked_batch_loader is not None: + # 批量读 + 批量物化：每段一次 mx.array，写池每 key 一次 scatter（碎 kernel 最少） + stacked = self.stacked_batch_loader(layer, misses) + self._place_experts_stacked(layer, misses, stacked, current=current) + else: + if self.batch_loader is not None: + # 一次并行读本层所有 miss（8-worker pread）；保序写池，LRU/驱逐语义不变 + loaded = self.batch_loader(layer, misses) + else: + loaded = {e: self.loader(layer, e) for e in misses} + # 批量写：每 key 仅一次 stacked scatter，取代逐专家逐段碎 scatter + self._place_experts(layer, misses, loaded, current=current) + # slots 与原始 expert_ids 一一对应(含重复)，便于直接 reshape 成 routing 索引 + slots = [slot_of[int(e)] for e in expert_ids] + return self._pools[layer], slots + + def _set_table(self, layer: int, e: int, slot: int): + t = self._slot_table.get(layer) + if t is not None: + t[int(e)] = slot # (num_experts,) 小张量原地改,代价可忽略 + + def _clear_table(self, layer: int, e: int): + t = self._slot_table.get(layer) + if t is not None: + t[int(e)] = -1 + + def _ensure_table(self, layer: int, num_experts: int) -> mx.array: + t = self._slot_table.get(layer) + if t is None or int(t.shape[0]) != num_experts: + self._ensure_layer(layer) + tab = [-1] * num_experts + for e, slot in self._slot_of[layer].items(): + tab[int(e)] = int(slot) + t = mx.array(tab, dtype=mx.int32) + self._slot_table[layer] = t + return t + + def acquire_gpu(self, layer: int, inds: mx.array, num_experts: int): + """GPU 侧 slot 重映射:命中层零 host 往返(仅一次 miss 标志同步)。 + + 返回 (pool_arrays, local)。inds 为路由结果(decode 时形如 (1,1,k))。 + 全命中 → local = 表[inds] 纯 GPU;有 miss → 回退既有 host acquire 读盘并维护表后重算 local。 + """ + table = self._ensure_table(layer, num_experts) + local = mx.take(table, inds) + n_miss = int(mx.sum((local < 0).astype(mx.int32))) # 唯一一次 GPU→CPU 同步 + if n_miss == 0: + self.gpu_fastpath += 1 + self.hits += int(inds.size) # decode top-k 各专家互异 → size 即唯一命中数 + if self.eviction_policy == "lfu": # 全命中层也计频:piggyback 上面 n_miss 的 drain,inds 已物化 + flat = [int(i) for i in inds.reshape(-1).tolist()] + self._note_access(layer, flat) + autopin.note(layer, flat) # AUTOPIN 热度:搭同一物化点,零额外同步(AUTOPIN=0 立即返回) + return self._pools[layer], local + # 有 miss:回退 host 路径(读盘、写槽、维护表),再用更新后的表重算 local + self.gpu_fallback += 1 + flat = [int(i) for i in inds.reshape(-1).tolist()] + autopin.note(layer, flat) # AUTOPIN 热度:miss 回退 flat 已物化,顺带计频 + pool_arrays, _ = self.acquire(layer, flat) + local = mx.take(self._slot_table[layer], inds) + return pool_arrays, local + + def verify_acquire_bytes(self, layer, inds, stg=None): + """诊断(STG_VERIFY)：acquire 后把本层真实路由命中专家的池槽字节与磁盘真值逐 key 比对。 + + 发现不一致即打印 (call, layer, expert, slot, gen, key)：池槽装错字节的铁证， + 与 timing/gen 竞态对应。call 是全局校验调用序号(≈token 前向序，便于和首分歧 token 关联)。 + stg：可选 NativeStagingManager，用于查该专家落池所用 gen。 + """ + st = _stg_verify_state + st["calls"] += 1 + call = st["calls"] + pool = self._pools.get(layer) + slot_of = self._slot_of.get(layer) + if pool is None or slot_of is None: + return + flat = {int(i) for i in inds.reshape(-1).tolist()} + for e in flat: + slot = slot_of.get(e) + if slot is None: + continue + try: + truth = self.loader(layer, e) + except Exception: + continue + bad_key = None + for k in pool: + if k not in truth: + continue + a = pool[k][slot] + b = truth[k] + if a.shape != b.shape or not bool(mx.all(a == b)): + bad_key = k + break + if bad_key is None: + st["ok"] += 1 + else: + st["bad"] += 1 + if st["first_bad_call"] is None: + st["first_bad_call"] = call + gen = None + if stg is not None: + gen = stg.placed_gen.get((layer, e)) + if st["printed"] < 24: + st["printed"] += 1 + print(f"[STG_VERIFY] BAD call={call} layer={layer} expert={e} " + f"slot={slot} gen={gen} key={bad_key} " + f"(ok={st['ok']} bad={st['bad']})", flush=True) + + def hit_rate(self) -> float: + tot = self.hits + self.misses + return self.hits / tot if tot else 0.0 diff --git a/mlx_streaming/core/cache/virtual_pool.py b/mlx_streaming/core/cache/virtual_pool.py new file mode 100644 index 0000000..d1734fa --- /dev/null +++ b/mlx_streaming/core/cache/virtual_pool.py @@ -0,0 +1,237 @@ +"""VirtualPool：预取调度器 + 双源双缓冲协调器（统一收口）。 + +一个对象承担两职，因为 block.py 用同一个 `self._vpool` 属性： + +1. ahead 调度（两种模式都用）：`ahead_for` / `target_for` 决定「第 L 层应预读哪层」—— + 早层用小 ahead 保召回、cutoff 起用大 ahead 抢时序。`_native_fused_prefetch` 靠它选目标层。 + +2. 双源双缓冲协调（仅 ZEROCOPY_DUAL_SOURCE 模式）：`begin_forward` / `read_gen` / + `fill_gen` / `acquire` / `prefetch`。侧区分两「代(gen)」物理行，每前向读上一代、fill 写 + 另一代，根除「本前向消费 gather 读的物理行在本前向 eval 期被 fill 覆盖」的竞态 + （见 spec 2026-06-25-qwen-virtual-pool-double-buffer）。对外只暴露「专家→物理行」单次 + gather 接口，消费者每层零 host 同步。 + +构造两种签名并存（互斥使用）： +- 调度器：VirtualPool(num_layers=.., cutoff=.., ahead_lo=.., ahead_hi=..) +- 协调器：VirtualPool(resident, staging, spec_slots)，dual-source 下再补调度参数即可两职合一。 +""" +import mlx.core as mx + +from mlx_streaming.core.cache import autopin + +# 方案B STG_VERIFY 校验累计态（诊断用，默认路径不触及）。 +_stg_verify_state = {"ok": 0, "bad": 0, "printed": 0, "calls": 0} + + +class VirtualPool: + def __init__(self, resident=None, staging=None, spec_slots=None, *, + num_layers=None, cutoff=None, ahead_lo=None, ahead_hi=None, store=None): + # --- 双源协调（resident/staging 存在时启用）--- + self._rp = resident + self._stg = staging + self._store = store # acquire_host 超容量 fetch 回退用(blob 感知);调度器形态可为 None + self._spec = int(spec_slots) if spec_slots is not None else 0 + self._gen = 0 + self._last_layer = -1 # 前向边界检测：层号回绕(<= 上次) 即新前向 + # --- ahead 调度 --- + self._num_layers = int(num_layers) if num_layers is not None else 0 + self._cutoff = int(cutoff) if cutoff is not None else 0 + self._a_lo = max(1, int(ahead_lo)) if ahead_lo is not None else 1 + self._a_hi = max(1, int(ahead_hi)) if ahead_hi is not None else 1 + + # ---- ahead 调度 ---- + def ahead_for(self, src_layer: int) -> int: + # cutoff：早层用 lo（保召回），cutoff 起用 hi（保时序）。 + return self._a_lo if int(src_layer) < self._cutoff else self._a_hi + + def target_for(self, src_layer: int) -> int: + # 目标层 = src+ahead。末层无可预读 → 返回 0（跳过）。 + # 不再 clamp 到末层：clamp 会让多个源层同时预读同一末层，对该层每前向 submit 次数 > + # staging ring，其环形 buffer 在惰性切片被本前向 eval 消费前就被后续 submit 完成回调覆盖成 + # 别的专家字节 → 池槽装错字节。越界即跳过（交给 demand 读真值），保证每层每前向至多 1 次 submit。 + L = int(src_layer) + if L >= self._num_layers - 1: + return 0 + tgt = L + self.ahead_for(L) + if tgt > self._num_layers - 1: # 越界（本会触发 clamp 堆叠）→ 跳过 + return 0 + return tgt + + # ---- 双源双缓冲协调 ---- + def begin_forward(self, layer_idx: int): + """每个 MoE 块 __call__ 开头调；层号回绕(<= 上次) 判为新前向 → 代 +1。 + 稳健：不依赖首个 MoE 层是 layer 0、也不要求 MoE 层连续。""" + if layer_idx <= self._last_layer: + self._gen += 1 + # AUTOPIN:dual decode 无 Python 计频点,用前向边界驱动周期落盘(AUTOPIN=0 立即返回)。 + autopin.tick() + # 新前向开头排空上一前向提交的侧区 fill：C++-owned 池 buffer 由后台异步直写， + # 若消费前 fill 未写完，GPU gather 会读到半写侧区行（DUAL_VERIFY BAD）。 + # drain 阻塞到在途 fill 全部写完 → 本前向要消费的侧区行字节必已就绪。 + if self._spec > 0: + import mlx_streaming.native_moe_ext as _N + _N.sideregion_drain() + self._last_layer = layer_idx + + def _gens(self) -> int: + # 代数取自常驻池;单代(=1)时读=填=0(持久 LFU 单区),双代(=2)时交替(%2==&1)。 + g = getattr(self._rp, "spec_gens", 2) if self._rp is not None else 2 + return max(1, int(g)) + + def read_gen(self) -> int: + return (self._gen - 1) % self._gens() # 读上一前向填好的代;单代恒 0 + + def fill_gen(self) -> int: + return self._gen % self._gens() # fill 写本代;单代恒 0 + + def acquire(self, layer, inds, num_experts, *, seq_len=None, layer_cap=None): + """统一取用入口（GPU-remap 路径）：对外呈现「所有专家都在」的视角。 + + 返回 (pool_arrays, local, n_experts)，计算侧零分支： + - dual（有侧区 staging 且 spec>0）：C++ demand_dual 唯一权威（真实区 ∪ 侧区读代单次 gather）； + n_experts = layer_cap + spec_gens*spec_slots。 + - 非 dual GPU-remap：acquire_gpu；n_experts = layer_cap。 + （host/fetch 路径见 acquire_host，其输入是 host 侧已 .tolist 的 flat，语义不同故分开。） + """ + cap = int(layer_cap) if layer_cap is not None else self._rp.cap_for(layer) + if self._stg is not None and self._spec > 0: + side_gen = self.read_gen() + # C++ demand_dual 是双源 decode 真实区的唯一权威（每层 1 次 inds 同步、零主线程落池/记账）。 + # 无 Python 退路：native 未编译 demand_dual → 明确报错（decode 依赖 native 全套能力）。 + if not getattr(self._rp, "_native_demand", False): + raise RuntimeError( + "双源 decode 需要 native demand_dual（已成唯一权威），但未检出。请编译 native_moe_ext。") + pool, local = self._acquire_native(layer, inds, side_gen, cap) + n_exp = cap + self._rp.spec_gens * self._rp.spec_slots + return pool, local, n_exp + pool, local = self._rp.acquire_gpu(layer, inds, num_experts) + return pool, local, cap + + def _native_meta(self, layer): + """缓存每层 demand_dual 的不变入参（pool_list/seg_nbytes/path），避免逐层重建 host 胶水。""" + cache = getattr(self, "_nd_meta", None) + if cache is None: + cache = self._nd_meta = {} + m = cache.get(layer) + if m is None: + stg = self._stg + segs = stg.src._segs # (proj, tensor, dt, shape, nb)，与池 key 同序 + pool_list = [self._rp._pools[layer][f"{p}.{t}"] for p, t, *_ in segs] + m = (pool_list, [int(nb) for *_, nb in segs], + f"{stg.src.dir}/layer{int(layer):02d}.blob", int(stg.stride)) + cache[layer] = m + return m + + def _acquire_native(self, layer, inds, side_gen, cap): + """方案B 取用：委派 C++ demand_dual（每层 1 次 inds 同步 + 并行 worker pread 落池）， + 更新 rp 统计计数（供报告口径一致）。""" + import mlx_streaming.native_moe_ext as N + rp = self._rp + rp._bootstrap_dual_pool(layer) # 首次建池 + real_init（幂等） + # 方案B 容量前提：miss 只能落真实区的 cap 槽，其中 pinned 永不可驱逐 → 可安放上限 cap−pinned。 + # 超了 C++ 会把专家落 0 号槽（拿别的专家权重算）→ 逐位不正确。调用方（block.py 的分流判据） + # 负责在超容量前把该前向送去 host/fetch；这里只做一次性告警兜底。 + _placeable = int(cap) - int(N.real_pinned_count(int(layer))) + if int(inds.size) > _placeable and not getattr(self, "_overcap_warned", False): + self._overcap_warned = True + import sys + print(f"[DEMAND_DUAL] 警告：inds.size={int(inds.size)} > 可安放槽 {_placeable}" + f"(cap={int(cap)} − pinned={int(cap) - _placeable})，真实区超容量，逐位将不正确。" + f"请调小 PREFILL_CHUNK，或调大 EXPERT_SLOTS / 调小 AUTOPIN_BUDGET_FRAC。", + file=sys.stderr, flush=True) + pool_list, seg_nbytes, path, stride = self._native_meta(layer) + local = N.demand_dual(inds, pool_list, seg_nbytes, int(layer), int(side_gen), path, + stride, int(cap), + rp.eviction_policy == "lfu", int(rp.lfu_decay_interval)) + st = N.demand_last_stats() # [hitpos, misspos, loads, fallback01, unplaced] + rp.hits += st[0] + rp.misses += st[2] + if len(st) > 4: + rp.unplaced += st[4] # >0 即有前向落了 0 号槽，输出不可信（/api/stats 暴露） + if st[3] == 0: + rp.gpu_fastpath += 1 + else: + rp.gpu_fallback += 1 + from mlx_streaming import config + if config.stg_verify(): # 诊断:方案B 池字节逐 key 真值校验(默认关) + self._verify_native_bytes(layer, inds, local) + return rp._pools[layer], local + + def _verify_native_bytes(self, layer, inds, local): + """诊断(STG_VERIFY，方案B)：校验「字节落池不变量」——真实区每个占用槽的池字节 == 该槽当前 + C++ 属主专家(g_real)的 blob 真值。这是 C++ 接管落池的字节等价铁证；发现不一致即池装错字节。 + + 注：不以 local→expert 为判据（local 可能因跨调用/多模型共享 g_real 而滞后于 g_real，属路由级 + 问题、非落池字节问题；逐位权威信号是 e2e n_mismatch）。 + """ + st = _stg_verify_state + st["calls"] += 1 + pool = self._rp._pools.get(layer) + if pool is None: + return + import mlx_streaming.native_moe_ext as N + stg = self._stg + path = f"{stg.src.dir}/layer{int(layer):02d}.blob" + segs = stg.src._segs + flat = N.real_region_contents(int(layer)) # [expert0,slot0,expert1,slot1,...] + for j in range(0, len(flat), 2): + e, slot = flat[j], flat[j + 1] + raw = N.blob_load(path, mx.array([e], dtype=mx.uint32), int(stg.stride))[0] + bad, off = None, 0 + for p, t, dt, shape, nb in segs: + k = f"{p}.{t}" + pv = pool[k][slot].reshape(-1).view(mx.uint8) + if not bool(mx.all(pv == raw[off:off + nb])): + bad = k + break + off += nb + if bad is None: + st["ok"] += 1 + else: + st["bad"] += 1 + if st["printed"] < 12: + st["printed"] += 1 + print(f"[STG_VERIFY-DUAL] BAD 落池字节错 call={st['calls']} layer={layer} " + f"expert={e} slot={slot} key={bad} (ok={st['ok']} bad={st['bad']})", flush=True) + + def acquire_host(self, layer, flat, inds_shape, inds_dtype, layer_cap): + """host/fetch 路径收口（prefill/大 seq 或关 GPU-remap）：flat 为 host 侧路由 id 列表。 + + 返回 (pool_arrays, local, n_experts)，与 block.py 原 host/fetch 分支逐元素等价： + - uniq <= cap：acquire(flat)，local 为槽位；n_experts = layer_cap。 + - uniq > cap：fetch(uniq_sorted)，local 为 remap 到 [0,uniq) 连续索引；n_experts = uniq 数。 + + native demand(dual)下真实区槽状态归 C++ g_real，Python 的 _slot_of/_free 从不填充， + 走 self._rp.acquire 必然在 _choose_victim 抛 "no evictable non-current slot"。 + 故 dual 下一律走 fetch 旁路（不进池、正确但慢）——这条路径本就是超容量前向的兜底。 + """ + import mlx.core as mx + cap = int(layer_cap) + uniq_set = set(flat) + # 入池的充分条件是 uniq ≤ cap − pinned:acquire 把本次全部唯一专家列为不可驱逐(current), + # 腾槽时只能挑「非 pinned 且非 current」的受害者,少一个都会在 _choose_victim 抛 + # "no evictable non-current slot"。pinned 是那层实际钉死数,不是拍出来的固定余量—— + # 写死余量会在 cap 小于余量时把所有请求推进 fetch 慢路径(每次重读盘)。 + _pinned_n = len(getattr(self._rp, "_pinned", {}).get(layer, ()) or ()) + if (not getattr(self._rp, "_native_demand", False) + and len(uniq_set) <= max(1, cap - _pinned_n)): + pool, slots = self._rp.acquire(layer, flat) + local = mx.array(slots, dtype=inds_dtype).reshape(inds_shape) + return pool, local, cap + uniq_sorted = sorted(uniq_set) + remap = {g: i for i, g in enumerate(uniq_sorted)} + local = mx.array([remap[i] for i in flat], dtype=inds_dtype).reshape(inds_shape) + # 超容量(或 dual:真实区槽状态归 C++ g_real,Python _slot_of 从不填充 → acquire 必抛) + # 走 fetch 旁路:不进池、按 uniq 现堆一份参与计算,正确但慢。这条路径本就是兜底。 + # fetch 由 blob 感知的 FileExpertStore 提供;ResidentExpertPool 本身没有,故优先用注入的 store。 + src = self._store if self._store is not None else self._rp + if not hasattr(src, "fetch"): + raise RuntimeError("VirtualPool.acquire_host 超容量回退需要带 fetch 的 store(构造时传入)") + fetched = src.fetch(layer, uniq_sorted) + return fetched, local, len(uniq_sorted) + + def prefetch(self, layer, pred, resident, pool_list): + """向 fill 代 submit 预读：base_row = cap_for(layer) + fill_gen*spec。""" + g = self.fill_gen() + base = self._rp.cap_for(layer) + g * self._spec + return self._stg.submit_pool_sideregion(layer, pred, resident, pool_list, base, gen=g) diff --git a/mlx_streaming/core/linear_attn/__init__.py b/mlx_streaming/core/linear_attn/__init__.py new file mode 100644 index 0000000..ddda1f0 --- /dev/null +++ b/mlx_streaming/core/linear_attn/__init__.py @@ -0,0 +1 @@ +"""线性注意力（Qwen3-Next gated-delta）相关的自定义 kernel。""" diff --git a/mlx_streaming/core/linear_attn/gated_delta_multistate.py b/mlx_streaming/core/linear_attn/gated_delta_multistate.py new file mode 100644 index 0000000..634132c --- /dev/null +++ b/mlx_streaming/core/linear_attn/gated_delta_multistate.py @@ -0,0 +1,226 @@ +"""逐步状态输出版 gated-delta Metal kernel（vendored 自 mlx_lm.models.gated_delta）。 + +来源：mlx-lm 0.x `mlx_lm/models/gated_delta.py` 的 `_make_gated_delta_kernel` +（本机 mlx 0.31.2）。本文件**只**在mlx kernel 基础上多加一个输出 `states_out`： +在时间维 `for (t=0..T-1)` 的串行递归里，每算完一步就把当前 state 写回 +`states_out[B, T, Hv, Dv, Dk]`。 + +为什么 bit-exact：mlx kernel 的时间维本就是线程内串行递归（不是分块并行），第 t 步 +寄存器里的 `state` 就是「逐 token 走 t+1 步后的状态」，全程 fp32、同序、同融合。因此 +一次 seq=T 调用产出的 `states_out[:, t]`，与逐 token（T 次 seq=1）调用链得到的 state +逐 bit 相等。这正是 MTP 投机验证后能「按接受长度直接提交、零 replay」的数值根基。 + +用途：MTP 验证前向里捕获每个 token 后的 ssm 递归态作为 checkpoint，替换原先走 +`_gated_delta_step_ops`（与 baseline kernel 不 bit-exact）的捕获路径。 + +非目标：仅 Metal(GPU) 路径；CPU/ops 回退不在此实现（部署恒为 Metal）。 +""" +from typing import Optional, Tuple + +import mlx.core as mx +import mlx.nn as nn + +# 与 mlx 一致：g = exp(-exp(A_log) * softplus(a + dt_bias))，fp32。 +from mlx_lm.models.gated_delta import compute_g + + +def _make_gated_delta_multistate_kernel(has_mask=False, vectorized=False): + """构造逐步状态输出版 kernel；除多写 states_out 外与mlx kernel 完全一致。""" + if not mx.metal.is_available(): + return None + mask_source = "mask[b_idx * T + t]" if has_mask else "true" + + # g 索引方式：标量门控 [B,T,Hv] vs 向量门控 [B,T,Hv,Dk]，与 mlx 对齐。 + if vectorized: + g_comment = "// g: [B, T, Hv, Dk]" + g_setup = "auto g_ = g + (b_idx * T * Hv + hv_idx) * Dk;" + g_access = "g_[s_idx]" + g_advance = "g_ += Hv * Dk;" + else: + g_comment = "// g: [B, T, Hv]" + g_setup = "auto g_ = g + b_idx * T * Hv;" + g_access = "g_[hv_idx]" + g_advance = "g_ += Hv;" + + # 与 mlx 版唯一的差异：时间循环每步末把 state 写入 states_out（见下方标注的新增块）。 + source = f""" + auto n = thread_position_in_grid.z; + auto b_idx = n / Hv; + auto hv_idx = n % Hv; + auto hk_idx = hv_idx / (Hv / Hk); + constexpr int n_per_t = Dk / 32; + + // q, k: [B, T, Hk, Dk] + auto q_ = q + b_idx * T * Hk * Dk + hk_idx * Dk; + auto k_ = k + b_idx * T * Hk * Dk + hk_idx * Dk; + + // v, y: [B, T, Hv, Dv] + auto v_ = v + b_idx * T * Hv * Dv + hv_idx * Dv; + y += b_idx * T * Hv * Dv + hv_idx * Dv; + + auto dk_idx = thread_position_in_threadgroup.x; + auto dv_idx = thread_position_in_grid.y; + + // state_in, state_out: [B, Hv, Dv, Dk] + auto i_state = state_in + (n * Dv + dv_idx) * Dk; + auto o_state = state_out + (n * Dv + dv_idx) * Dk; + + float state[n_per_t]; + for (int i = 0; i < n_per_t; ++i) {{ + auto s_idx = n_per_t * dk_idx + i; + state[i] = static_cast(i_state[s_idx]); + }} + + {g_comment} + {g_setup} + auto beta_ = beta + b_idx * T * Hv; + + for (int t = 0; t < T; ++t) {{ + if ({mask_source}) {{ + float kv_mem = 0.0f; + for (int i = 0; i < n_per_t; ++i) {{ + auto s_idx = n_per_t * dk_idx + i; + state[i] = state[i] * {g_access}; + kv_mem += state[i] * k_[s_idx]; + }} + kv_mem = simd_sum(kv_mem); + + auto delta = (v_[dv_idx] - kv_mem) * beta_[hv_idx]; + + float out = 0.0f; + for (int i = 0; i < n_per_t; ++i) {{ + auto s_idx = n_per_t * dk_idx + i; + state[i] = state[i] + k_[s_idx] * delta; + out += state[i] * q_[s_idx]; + }} + out = simd_sum(out); + if (thread_index_in_simdgroup == 0) {{ + y[dv_idx] = static_cast(out); + }} + }} else {{ + y[dv_idx] = static_cast(0); + }} + + // ===== 新增：每步把当前 state 写回 states_out[B, T, Hv, Dv, Dk] ===== + // masked 步 state 不变（上面 else 分支不更新 state），写回的就是上一步的状态， + // 与逐 token 单步链语义一致。 + auto ms_ = states_out + + ((((b_idx * T + t) * Hv + hv_idx) * Dv + dv_idx) * Dk); + for (int i = 0; i < n_per_t; ++i) {{ + auto s_idx = n_per_t * dk_idx + i; + ms_[s_idx] = static_cast(state[i]); + }} + // =================================================================== + + // Increment data pointers to next time step + q_ += Hk * Dk; + k_ += Hk * Dk; + v_ += Hv * Dv; + y += Hv * Dv; + {g_advance} + beta_ += Hv; + }} + for (int i = 0; i < n_per_t; ++i) {{ + auto s_idx = n_per_t * dk_idx + i; + o_state[s_idx] = static_cast(state[i]); + }} + """ + inputs = ["q", "k", "v", "g", "beta", "state_in", "T"] + if has_mask: + inputs.append("mask") + + suffix = "" + if vectorized: + suffix += "_vec" + if has_mask: + suffix += "_mask" + + return mx.fast.metal_kernel( + name=f"gated_delta_step_multistate{suffix}", + input_names=inputs, + output_names=["y", "state_out", "states_out"], + source=source, + ) + + +_kernel = _make_gated_delta_multistate_kernel(has_mask=False, vectorized=False) +_kernel_masked = _make_gated_delta_multistate_kernel(has_mask=True, vectorized=False) +_kernel_vec = _make_gated_delta_multistate_kernel(has_mask=False, vectorized=True) +_kernel_vec_masked = _make_gated_delta_multistate_kernel(has_mask=True, vectorized=True) + + +def gated_delta_multistate_kernel( + q: mx.array, + k: mx.array, + v: mx.array, + g: mx.array, + beta: mx.array, + state: mx.array, + mask: Optional[mx.array] = None, +) -> Tuple[mx.array, mx.array, mx.array]: + """逐步状态输出版 kernel 调用，返回 (y, final_state, states_out)。 + + 形状（与 mlx `gated_delta_kernel` 一致，不在 Python 侧做 GQA repeat）： + - q, k: [B, T, Hk, Dk] + - v: [B, T, Hv, Dv] + - g: [B, T, Hv]（标量）或 [B, T, Hv, Dk]（向量） + - beta: [B, T, Hv] + - state:[B, Hv, Dv, Dk] + 返回： + - y: [B, T, Hv, Dv] + - final_state:[B, Hv, Dv, Dk]（== states_out[:, -1]） + - states_out: [B, T, Hv, Dv, Dk]（每个 token 处理完后的递归态） + """ + B, T, Hk, Dk = k.shape + Hv, Dv = v.shape[2:] + input_type = q.dtype + state_type = state.dtype + if g.ndim == 4: + kernel = _kernel_vec_masked if mask is not None else _kernel_vec + else: + kernel = _kernel_masked if mask is not None else _kernel + + inputs = [q, k, v, g, beta, state, T] + if mask is not None: + inputs.append(mask) + + return kernel( + inputs=inputs, + template=[ + ("InT", input_type), + ("StT", state_type), + ("Dk", Dk), + ("Dv", Dv), + ("Hk", Hk), + ("Hv", Hv), + ], + grid=(32, Dv, B * Hv), + threadgroup=(32, 4, 1), + output_shapes=[(B, T, Hv, Dv), state.shape, (B, T, Hv, Dv, Dk)], + output_dtypes=[input_type, state_type, state_type], + ) + + +def gated_delta_update_multistate( + q: mx.array, + k: mx.array, + v: mx.array, + a: mx.array, + b: mx.array, + A_log: mx.array, + dt_bias: mx.array, + state: Optional[mx.array] = None, + mask: Optional[mx.array] = None, +) -> Tuple[mx.array, mx.array, mx.array]: + """对齐 mlx `gated_delta_update` 的签名，多返回逐步状态 states_out。 + + 内部 beta/g 计算与 mlx 完全一致（sigmoid(b) / compute_g），保证数值路径相同。 + 仅走 Metal kernel 路径（multistate 仅 GPU 实现）。 + """ + beta = mx.sigmoid(b) + g = compute_g(A_log, a, dt_bias) + if state is None: + B, _, Hk, Dk = q.shape + Hv, Dv = v.shape[-2:] + state = mx.zeros((B, Hv, Dv, Dk), dtype=mx.float32) + return gated_delta_multistate_kernel(q, k, v, g, beta, state, mask) diff --git a/mlx_streaming/core/mem.py b/mlx_streaming/core/mem.py new file mode 100644 index 0000000..4d8eec3 --- /dev/null +++ b/mlx_streaming/core/mem.py @@ -0,0 +1,75 @@ +"""内存度量统一口径。macOS 上 ru_maxrss 单位是字节。""" +import resource +from dataclasses import dataclass + +import mlx.core as mx + + +def rss_bytes() -> int: + # macOS: ru_maxrss 已是字节；Linux 是 KB（本项目目标是 macOS） + return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + + +def _call(name: str) -> int: + fn = getattr(mx, name, None) + if fn is None: + fn = getattr(getattr(mx, "metal", object()), name, None) + try: + return int(fn()) if fn else 0 + except Exception: + return 0 + + +@dataclass +class MemSnapshot: + rss_bytes: int + mlx_active_bytes: int + mlx_peak_bytes: int + + +def snapshot() -> MemSnapshot: + return MemSnapshot( + rss_bytes=rss_bytes(), + mlx_active_bytes=_call("get_active_memory"), + mlx_peak_bytes=_call("get_peak_memory"), + ) + + +def clear_cache() -> None: + fn = getattr(mx, "clear_cache", None) or getattr(getattr(mx, "metal", object()), "clear_cache", None) + if fn: + fn() + + +def reset_peak() -> None: + fn = getattr(mx, "reset_peak_memory", None) or getattr(getattr(mx, "metal", object()), "reset_peak_memory", None) + if fn: + fn() + + +def _set_limit(name: str, nbytes: int) -> bool: + """调用 mx.set_*_limit(字节);存在且成功返回 True。""" + fn = getattr(mx, name, None) or getattr(getattr(mx, "metal", object()), name, None) + if fn is None: + return False + try: + fn(int(nbytes)) + return True + except Exception: + return False + + +def setup_memory_hygiene(cache_gb: float = 2.0, wired_gb: float = 0.0) -> dict: + """长期运行的内存防御:bound MLX 缓冲缓存 + 可选 wire 工作集防 macOS 压缩器。 + + - cache_gb>0:set_cache_limit,封顶 MLX 可回收缓冲,防长会话里缓存膨胀把常驻推过墙。 + - wired_gb>0:set_wired_limit,把这么多 GB 的 GPU 缓冲钉为常驻(wired),macOS 不再 + 压缩/换出这些页 → 长跑延迟稳定。务必 < 系统建议工作集(本机 26.8GB),否则饿死系统。 + 返回实际生效项,便于启动日志记录。 + """ + applied = {} + if cache_gb and cache_gb > 0: + applied["cache_limit_gb"] = cache_gb if _set_limit("set_cache_limit", cache_gb * 1e9) else None + if wired_gb and wired_gb > 0: + applied["wired_limit_gb"] = wired_gb if _set_limit("set_wired_limit", wired_gb * 1e9) else None + return applied diff --git a/mlx_streaming/core/moe/__init__.py b/mlx_streaming/core/moe/__init__.py new file mode 100644 index 0000000..cdbd4e3 --- /dev/null +++ b/mlx_streaming/core/moe/__init__.py @@ -0,0 +1 @@ +"""MoE 模块：门控选专家(gate) / 专家计算(compute) / 自定义算子(custom_kernel) / 热路径块(block)。""" diff --git a/mlx_streaming/core/moe/block.py b/mlx_streaming/core/moe/block.py new file mode 100644 index 0000000..bf6850c --- /dev/null +++ b/mlx_streaming/core/moe/block.py @@ -0,0 +1,431 @@ +"""MoE 热路径块：路由 → 选专家 → 取专家权重 → 计算 → 加权合并。 + +包含两种块： +- `StreamingMoeBlock`：包住原生 MoE 块，专家权重常驻（switch_mlp），只激活选中专家。 +- `FileStreamingMoeBlock`：专家权重从磁盘按需加载（流式），是低内存推理的核心热路径， + 集成常驻池 acquire、native-fused-prefetch 预取、共享专家叠加等逻辑。 +""" +import os +import time + +import mlx.core as mx + +from mlx_streaming import config +from mlx_streaming.core.moe import native_moe +from mlx_streaming.core import route_trace +from mlx_streaming.core.profiling import ( + PROF, WINDOW_PROF, PREDICT_RECALL_PROF, MISS_ATTRIB, note_miss_attrib, + note_tprof, TPROF_ON, _PROF_ON, _tick, note_union, UNION_ON) + +# decode/verify 热路径判据:seq 短(单 token decode=1、MTP verify=K≤几)，与 prefill 长 seq 区分。 +_DECODE_SEQ_MAX = 8 +from mlx_streaming.core.moe.gate import _effective_top_k +from mlx_streaming.core.cache import autopin +from mlx_streaming.core.moe.compute import ( + streaming_switch_glu_forward, PersistentSubGLU) + + +class StreamingMoeBlock: + """包住原 Qwen3MoeSparseMoeBlock：路由器常驻，专家计算改为只算选中专家。 + + 若提供 store（LruExpertStore），可进一步把专家权重从磁盘按需加载；否则直接在 + 常驻的 switch_mlp 上做 uniq 切片计算（仍只激活少数专家）。 + """ + + def __init__(self, orig_block, layer_idx: int, store=None): + self.gate = orig_block.gate # 路由器常驻（很小） + self.top_k = orig_block.top_k + self.norm_topk_prob = orig_block.norm_topk_prob + self.switch_mlp = orig_block.switch_mlp + self.store = store + self.layer_idx = layer_idx + + def __call__(self, x: mx.array) -> mx.array: + gates = mx.softmax(self.gate(x), axis=-1, precise=True) + k = _effective_top_k(self.top_k) + inds = mx.argpartition(gates, kth=-k, axis=-1)[..., -k:] + scores = mx.take_along_axis(gates, inds, axis=-1) + if self.norm_topk_prob: + scores = scores / mx.sum(scores, axis=-1, keepdims=True) + mx.eval(inds) # 先物化路由结果，才能按结果取专家 + y = streaming_switch_glu_forward(self.switch_mlp, x, inds) + return (y * scores[..., None]).sum(axis=-2) + + +class FileStreamingMoeBlock: + """文件后端流式 MoE 块：路由器常驻，专家权重从磁盘按需加载（不持有堆叠 switch_mlp）。""" + + def __init__(self, gate, top_k, norm_topk_prob, store, layer_idx, + hidden, moe_inter, group_size, bits, + proj_bits: dict | None = None, + shared_expert=None, shared_expert_gate=None): + self.gate = gate + self.top_k = top_k + self.norm_topk_prob = norm_topk_prob + self.store = store + self.layer_idx = layer_idx + self.hidden = hidden + self.moe_inter = moe_inter + self.group_size = group_size + self.bits = bits + # Qwen3-Next 等带共享专家的模型：共享专家恒激活、必须常驻（不流式）， + # 输出叠加 sigmoid(shared_expert_gate(x)) * shared_expert(x)。None 则退化为纯路由（如 Qwen3-MoE）。 + self.shared_expert = shared_expert + self.shared_expert_gate = shared_expert_gate + # 可选全流式 blob 源（STREAM_BLOB=1 时由 model_builder 注入），见 core/blob_loader.py。 + self._blob = None + # 持久化子模块：跨 token 复用，避免每调用重建 QSL。 + # proj_bits 非空时走混合精度（逐 proj 不同 bit）。 + self._sub = PersistentSubGLU(hidden, moe_inter, group_size, bits, + proj_bits=proj_bits, layer_idx=layer_idx) + + def __call__(self, x: mx.array) -> mx.array: + if _PROF_ON: + return self._call_prof(x) + gates = mx.softmax(self.gate(x), axis=-1, precise=True) + k = _effective_top_k(self.top_k) + inds = mx.argpartition(gates, kth=-k, axis=-1)[..., -k:] + scores = mx.take_along_axis(gates, inds, axis=-1) + if self.norm_topk_prob: + scores = scores / mx.sum(scores, axis=-1, keepdims=True) + if config.predict_recall_prof(): + ps = getattr(self, "_predicted_set", None) + if ps is not None: + act = {int(i) for i in inds.reshape(-1).tolist()} + PREDICT_RECALL_PROF["hit"] += len(ps & act) + PREDICT_RECALL_PROF["routed"] += len(act) + PREDICT_RECALL_PROF["n"] += 1 + if config.native_moe(): + native_y = self._try_native_forward(x, inds, scores, gates.shape[-1]) + if native_y is not None: + y = native_y + if self.shared_expert is not None: + y = y + mx.sigmoid(self.shared_expert_gate(x)) * self.shared_expert(x) + return y + if self._blob is not None and config.stream_blob(): + # 全流式 blob 路径：每层按需并行读专家 → 复用 _sub.forward(MLX quantized_matmul)。 + flat = [int(i) for i in inds.reshape(-1).tolist()] + pool_arrays, slots = self._blob.acquire(self.layer_idx, flat) + local = mx.array(slots, dtype=inds.dtype).reshape(inds.shape) + y = self._sub.forward(pool_arrays, len(set(flat)), x, local) + y = (y * scores[..., None]).sum(axis=-2) + if self.shared_expert is not None: + y = y + mx.sigmoid(self.shared_expert_gate(x)) * self.shared_expert(x) + return y + if config.probe_perlayer_sync(): + # 诊断：每层强制一次 host 同步（模拟预测 barrier），不做任何预取。 + # 若单这个就拖慢 ~30% → 税是 eval barrier，C++ 读统一内存也救不了。 + _ = int(inds.reshape(-1)[:1].item()) + if (config.stream_blob_bg() + and getattr(self.store, "_bg", None) is not None): + # 后台预取已物化的专家在此（主线程、acquire 前）写进常驻池槽 → 转 miss 为 hit。 + if config.window_prof(): + t0 = getattr(self, "_submit_t", None) + if t0 is not None: + WINDOW_PROF["sum_s"] += time.perf_counter() - t0 + WINDOW_PROF["n"] += 1 + self.store.promote_prefetched(self.layer_idx) + # native-fused-prefetch 的 promote（把回调预读好的专家写进池槽）下移到各 acquire 分支前： + # host/verify 路径已算出真实路由 uniq_set，可"只 promote 命中本层路由的专家"，零额外同步地 + # 丢弃假阳性 → 省掉无用 scatter + 池污染（这是 promote -3.2% 开销的主因）。 + _stg_mgr = getattr(self.store, "_staging", None) + # 双源：投机专家留池侧区由 demand_dual 取，不 promote、不驱逐 → 关 promote。 + _do_promote = (_stg_mgr is not None and not config.native_no_promote() + and not config.zerocopy_dual_source()) + if config.route_trace_enabled(): + flat_trace = [int(i) for i in inds.reshape(-1).tolist()] + resident = (self.store.resident_experts(self.layer_idx) + if hasattr(self.store, "resident_experts") else set()) + rank = (self.store.resident_lru_scores(self.layer_idx) + if hasattr(self.store, "resident_lru_scores") else {}) + route_trace.record( + self.layer_idx, flat_trace, set(flat_trace) - resident, resident, rank) + # native-fused-prefetch：在 seq 分支**之前**提交，使 decode(seq=1) 与 MTP verify(seq=K) + # 两条路径都触发预取。dummy 折进 inds(加 0)：GPU 路径靠 acquire_gpu 的 n_miss eval、 + # host 路径靠后面的 .tolist() eval，都会触发完成回调里的 pread。verify 时 x 为 K 个 + # token（verify_in=[x, d_1..d_{K-1}]），预测的是"下一层这 K 个 token 的专家并集"（recall≈0.96 的口径）。 + # 双源双缓冲：每块开头先推进/检测前向边界（必须在本块预取提交之前，保证同前向内 fill_gen/read_gen 恒定）。 + if config.zerocopy_dual_source() and getattr(self, "_vpool", None) is not None: + self._vpool.begin_forward(self.layer_idx) + if (getattr(self.store, "_staging", None) is not None + and not config.native_no_submit()): + _dummy = self._native_fused_prefetch(x) + if _dummy is not None: + inds = inds + (_dummy.reshape(()).astype(inds.dtype) * 0) + layer_cap = self.store.cap_for(self.layer_idx) + # verify(小 seq)可走 GPU 重映射;prefill(大 seq)唯一专家可能超 cap,须留在 host 路径(有超容量 fetch 回退)。 + if config.zerocopy_dual_source() and getattr(self, "_vpool", None) is not None: + # 双源两级缓存本就是为 MTP verify 建的:verify 走 dual 路径才能读到侧区,否则落 host + # 路径会白填侧区(预取填了却不读)→ 命中骤降、读盘翻倍。 + # + # 但判据必须按 demand_dual 真实区**可安放容量**算,不能按 cap+侧区行:侧区行是预取只读的, + # miss 只能落真实区的 cap 槽,其中 pinned(AUTOPIN)永不可驱逐。设本次唯一专家数 U、 + # 真实区 cap=C、pinned=P,可用槽 = C − P − (本次命中的非 pinned 常驻数),推得 + # 「U ≤ C − P」是安放得下的充分条件。超了 C++ 分不到槽会把专家落 0 号槽 → 逐位错算 + # (prefill chunk 大时曾整段 system+tools 都算错)。U 无同步上界即 inds.size = seq×top_k。 + _verify_gpu = (x.shape[1] * k <= self._dual_placeable(layer_cap)) + else: + _verify_gpu = (config.verify_gpu_remap() and x.shape[1] * k <= layer_cap) + if (config.resident_pool_enabled() + and config.gpu_remap_enabled() + and (x.shape[1] == 1 or _verify_gpu)): + # decode 热路径:GPU 侧 slot 重映射,命中层零 host 往返(消除每层 .tolist 同步), + # 仅一次 miss 标志同步;真 miss 才回退读盘。decode top-k(≤cap)恒装得下。 + # 专家总数 = gate 输出维(gates 末维),无需依赖 gate.weight。 + if UNION_ON: + # GPU 路径本不 .tolist 真实路由;仅 UNION_PROF=1 时付一次同步取并集大小。 + note_union(x.shape[1], len({int(i) for i in inds.reshape(-1).tolist()}), + self.layer_idx) + # decode/verify GPU 重映射路径:host 无现成真实路由。开关开时用 GPU membership 仅对 + # 预读候选现算 used(drain≤budget)过滤假阳性;关时退回整批写入(回退/对照基线)。 + if _do_promote: + if config.gpu_remap_promote_filter(): + _stg_mgr.promote(self.layer_idx, self.store, + route_inds=inds, num_experts=gates.shape[-1]) + else: + _stg_mgr.promote(self.layer_idx, self.store) + if config.miss_attrib(): + # 诊断专用:GPU remap 路径本不 .tolist 真实路由(那是它消栅栏的关键), + # 仅 MISS_ATTRIB=1 时付一次 .tolist 取真实路由,统计 decode 热路径 A/B 构成。 + _uniq = {int(i) for i in inds.reshape(-1).tolist()} + _ps = getattr(self, "_predicted_set", None) + _res = (self.store.resident_experts(self.layer_idx) + if hasattr(self.store, "resident_experts") else set()) + _rdy = _stg_mgr.last_ready.get(self.layer_idx) if _stg_mgr else None + note_miss_attrib(_uniq, _ps, _res, x.shape[1] <= _DECODE_SEQ_MAX, _rdy) + if config.zerocopy_dual_source(): + # 双源双缓冲：统一走 VirtualPool.acquire（呈现「所有专家都在」视角， + # 内部真实区表 ∪ 侧区(读代) 单次 gather，返回 (pool, local, n_experts)）。 + pool_arrays, local, n_experts = self._vpool.acquire( + self.layer_idx, inds, gates.shape[-1], + seq_len=x.shape[1], layer_cap=layer_cap) + y = self._sub.forward(pool_arrays, n_experts, x, local) + else: + pool_arrays, local = self.store.acquire_gpu( + self.layer_idx, inds, gates.shape[-1]) + if config.stg_verify(): + # 诊断：消费侧字节校验（默认 off），定位混合 ahead 下池槽损坏。 + self.store._resident.verify_acquire_bytes( + self.layer_idx, inds, _stg_mgr) + y = self._sub.forward(pool_arrays, layer_cap, x, local) + else: + # prefill/大批量(seq>1)或显式关闭 GPU_REMAP:走 host 路径。 + # 只在此处对 inds 做一次 .tolist() 同步，uniq/local 全在 Python 里算。 + flat = [int(i) for i in inds.reshape(-1).tolist()] + autopin.note(self.layer_idx, flat) # AUTOPIN 热度:搭已物化的 flat,零额外同步(AUTOPIN=0 立即返回) + uniq_set = set(flat) + if UNION_ON: + note_union(x.shape[1], len(uniq_set), self.layer_idx) # 本层路由专家并集(零额外同步,uniq_set 已算) + # promote 只写"预读好 ∩ 本层真实路由"的专家：复用已算的 uniq_set，零额外同步， + # 假阳性不进池 → 省 scatter、不污染池（acquire 前完成，命中转化生效）。 + if _do_promote: + _stg_mgr.promote(self.layer_idx, self.store, used=uniq_set) + if config.miss_attrib(): + # 在 promote 之后、acquire(读盘)之前统计：本层真实路由的命中构成。 + # miss 拆成 A(预测到却没进池：budget丢/时序/驱逐) 与 B(没预测到：召回缺口)。 + _ps = getattr(self, "_predicted_set", None) + _res = (self.store.resident_experts(self.layer_idx) + if hasattr(self.store, "resident_experts") else set()) + _rdy = _stg_mgr.last_ready.get(self.layer_idx) if _stg_mgr else None + note_miss_attrib(uniq_set, _ps, _res, x.shape[1] <= _DECODE_SEQ_MAX, _rdy) + # 池只能同时容纳 ≤该层容量 个唯一专家；prefill 唯一数超容量时回退 stack。 + # dual 模式（_vpool 存在）统一走 VirtualPool.acquire_host 收口 host+fetch； + # 非 dual（_vpool 为 None）保留内联逻辑（非目标路径，避免额外依赖）。 + if (config.resident_pool_enabled() + and getattr(self, "_vpool", None) is not None): + pool_arrays, local, n_experts = self._vpool.acquire_host( + self.layer_idx, flat, inds.shape, inds.dtype, layer_cap) + y = self._sub.forward(pool_arrays, n_experts, x, local) + elif (config.resident_pool_enabled() + and len(uniq_set) <= layer_cap): + pool_arrays, slots = self.store.acquire(self.layer_idx, flat) + local = mx.array(slots, dtype=inds.dtype).reshape(inds.shape) + y = self._sub.forward(pool_arrays, layer_cap, x, local) + else: + uniq_sorted = sorted(uniq_set) + remap = {g: i for i, g in enumerate(uniq_sorted)} + local = mx.array([remap[i] for i in flat], dtype=inds.dtype).reshape(inds.shape) + fetched = self.store.fetch(self.layer_idx, uniq_sorted) + y = self._sub.forward(fetched, len(uniq_sorted), x, local) + y = (y * scores[..., None]).sum(axis=-2) + if self.shared_expert is not None: # 共享专家（常驻）叠加 + y = y + mx.sigmoid(self.shared_expert_gate(x)) * self.shared_expert(x) + return y + + def _dual_placeable(self, layer_cap: int) -> int: + """demand_dual 真实区本层能安放的唯一专家上限 = cap − pinned。 + + pinned 集合只在启动 AUTOPIN 预热时写入、生成期恒定,故首次查到后缓存 + (decode 每层每步都要判,不能每次都进 C++ 取锁)。 + """ + n = getattr(self, "_pinned_n", None) + if n is None: + try: + import mlx_streaming.native_moe_ext as _N + n = int(_N.real_pinned_count(int(self.layer_idx))) + except Exception: # noqa: BLE001 取不到就按最保守的「全钉死」算,逼回 host 路径 + n = int(layer_cap) + self._pinned_n = n + return max(1, int(layer_cap) - n) + + def _native_fused_prefetch(self, x: mx.array): + """搭车式预取：用下 AHEAD 层 gate 对 x 算预测 inds(lazy)，挂 GPU 完成回调， + 在 C++ 里(GPU 算完后)读 id + pread 预热下层专家字节。返回 dummy 张量(需被 eval 才触发)。 + 全程不加主线程 host 同步——靠折进 inds、搭 acquire_gpu 的 n_miss eval。 + """ + model = getattr(self, "_prefetch_model_ref", None) + bl = getattr(self.store, "_blob_loader", None) + if model is None or bl is None: + return None + layers = getattr(model, "layers", []) + vp = getattr(self, "_vpool", None) + if vp is not None: + tgt = vp.target_for(self.layer_idx) # per-layer cutoff ahead + if not (self.layer_idx < tgt < len(layers)): # 0/越界/无前瞻 → 跳过 + return None + else: + ahead = max(1, config.cross_layer_ahead(default=1)) + tgt = self.layer_idx + ahead + if not (0 <= tgt < len(layers)): + return None + tmlp = getattr(layers[tgt], "mlp", None) + if not isinstance(tmlp, FileStreamingMoeBlock): + return None + try: + from mlx_streaming import native_moe_ext as _N + except Exception: + return None + # 方案B：预测"宽集合"(top-N, N=predict_width)只为高 recall，不占内存（仅一次 gate argpartition）； + # 真正占 staging 的是 C++ 回调过滤常驻后、按分截到 staging budget 的"缺口"子集。 + _tp0 = time.perf_counter() if TPROF_ON else 0.0 + predict_width = config.cross_layer_predict_width() + # 默认(PREDICT_USE_X=1)：用本层 MoE 输入 x 喂目标层 gate。x 含了本层 attention，离 L+1 + # 近半层 → 比未归一化输入更新鲜，实测 recall 0.812→0.847(+3.6pp)、还省一次 norm。 + # PREDICT_USE_X=0 回退旧路径：目标层 post_attention_layernorm 作用在本层未归一化输入上。 + h = getattr(self, "_unnormed_input", None) + if config.predict_use_x(): + g = tmlp.gate(x) + elif h is None: + h = x # 退化：无未归一化输入时用 x（norm 偏差，覆盖会差） + g = tmlp.gate(h) + else: + g = tmlp.gate(layers[tgt].post_attention_layernorm(h)) + # 按 seq 维聚合成"该批 K 个 token 的专家并集"近似。聚合方式 PREDICT_AGG： + # - max（默认）：任一 token 强烈想要即入选；mean：各 token 平均偏好； + # - union：每 token 各取 top-kk 再并集（与真实路由"并集"结构最一致，候选更多，由 handler 去重+截断）。 + _agg = config.predict_agg() + kk = min(g.shape[-1], predict_width) + if g.ndim == 3 and _agg == "union": + # union 候选数 = seq × union_k。每 token 只取 top-union_k（独立于 predict_width）， + # 调小 union_k 缩候选 → 减少缺口、缓解 staging/读盘洪水（每 token 真实只路由 top_k 个）。 + uk = min(g.shape[-1], config.predict_union_k()) + pred = mx.argpartition(g, kth=-uk, axis=-1)[..., -uk:].reshape(-1).astype(mx.uint32) + else: + if g.ndim == 3: + g = g.mean(axis=1) if _agg == "mean" else g.max(axis=1) + # 取 top-kk（argpartition，O(E) 不全排序）。实测：加宽 kk 不提升命中——staging 更多 = 后台 + # pread 更多、到位更晚 + 抢带宽 → 反而更低；故 width=16 即最优，cap 截断极少触发、无需排序。 + pred = mx.argpartition(g, kth=-kk, axis=-1)[..., -kk:].reshape(-1).astype(mx.uint32) + if config.predict_recall_prof() or config.miss_attrib(): + try: + tmlp._predicted_set = {int(e) for e in pred.tolist()} + except Exception: + pass + if TPROF_ON: + note_tprof("predict_s", time.perf_counter() - _tp0, count_key="predict_n") + stg = getattr(self.store, "_staging", None) + if stg is not None: + resident = (self.store.resident_experts(tgt) + if hasattr(self.store, "resident_experts") else None) + if config.zerocopy_dual_source(): + # 双源双缓冲：经 VirtualPool 向填代 submit,C++ 回调把预读段散写进该代侧区行。 + rp = self.store._resident + if tgt not in rp._pools: + return None # 目标层池未建(首 token 预热) → 跳过本次 + segs = stg.src._segs # (proj, tensor, dt, shape, nb)，与池 key 顺序一致 + pool_list = [rp._pools[tgt][f"{p}.{t}"] for p, t, *_ in segs] + import os as _os + if _os.environ.get("POOL_PTR_TRACE") is not None and tgt == int(_os.environ["POOL_PTR_TRACE"]): + try: + import mlx_streaming.native_moe_ext as _N + _fk = f"{segs[0][0]}.{segs[0][1]}" + print(f"[POOL_PTR] submit layer={tgt} key={_fk} obj_id={id(rp._pools[tgt][_fk])} " + f"ptr={hex(_N.array_data_ptr(rp._pools[tgt][_fk]))}", flush=True) + except Exception as _e: + print(f"[POOL_PTR] err {_e}", flush=True) + return self._vpool.prefetch(tgt, pred, resident, pool_list) + # miss→hit:回调按目标层常驻快照过滤，只把缺口 pread 进 staging（≤budget 行），promote 时写池。 + if TPROF_ON: + _ts0 = time.perf_counter() + _r = stg.submit(tgt, pred, resident) + note_tprof("submit_s", time.perf_counter() - _ts0, count_key="submit_n") + return _r + return stg.submit(tgt, pred, resident) + # 仅预热字节（page cache）的轻量版。 + path = os.path.join(bl.dir, f"layer{tgt:02d}.blob") + return _N.prefetch_on_complete(pred, path, int(bl.stride), True) + + def _try_native_forward(self, x: mx.array, inds: mx.array, scores: mx.array, num_experts: int): + # native 第一版只支持三投影同 bit 的专家；其他情况保持现有 MLX 回退。 + proj_bits = getattr(self._sub, "proj_bits", {}) + bits_set = {int(proj_bits.get(name, self.bits)) for name in ("gate_proj", "up_proj", "down_proj")} + if len(bits_set) != 1: + return None + bits = bits_set.pop() + if not native_moe.can_native_moe( + self.layer_idx, self.hidden, self.moe_inter, + self.group_size, bits, num_experts): + return None + t_sync = time.perf_counter() + mx.eval(inds) + flat = [int(i) for i in inds.reshape(-1).tolist()] + native_moe.note_route_sync(time.perf_counter() - t_sync) + return native_moe.try_native_moe( + self.layer_idx, flat, x, scores, + self.hidden, self.moe_inter, self.group_size, bits, num_experts) + + def _call_prof(self, x: mx.array) -> mx.array: + t = time.perf_counter() + gates = mx.softmax(self.gate(x), axis=-1, precise=True) + k = _effective_top_k(self.top_k) + inds = mx.argpartition(gates, kth=-k, axis=-1)[..., -k:] + scores = mx.take_along_axis(gates, inds, axis=-1) + if self.norm_topk_prob: + scores = scores / mx.sum(scores, axis=-1, keepdims=True) + mx.eval(inds, scores) + _tick("route", t); t = time.perf_counter() + + flat = [int(i) for i in inds.reshape(-1).tolist()] + uniq_set = set(flat) + layer_cap = self.store.cap_for(self.layer_idx) + use_pool = (config.resident_pool_enabled() + and len(uniq_set) <= layer_cap) + if not use_pool: + uniq_sorted = sorted(uniq_set) + remap = {g: i for i, g in enumerate(uniq_sorted)} + local = mx.array([remap[i] for i in flat], dtype=inds.dtype).reshape(inds.shape) + mx.eval(local) + _tick("pyremap", t); t = time.perf_counter() + + if use_pool: + pool_arrays, slots = self.store.acquire(self.layer_idx, flat) + local = mx.array(slots, dtype=inds.dtype).reshape(inds.shape) + mx.eval(local) + fetched, n_experts = pool_arrays, layer_cap + else: + fetched = self.store.fetch(self.layer_idx, uniq_sorted) + n_experts = len(uniq_sorted) + mx.eval(list(fetched.values())) + _tick("fetch", t); t = time.perf_counter() + + y = self._sub.forward(fetched, n_experts, x, local) + mx.eval(y) + _tick("matmul", t); t = time.perf_counter() + + y = (y * scores[..., None]).sum(axis=-2) + if self.shared_expert is not None: + y = y + mx.sigmoid(self.shared_expert_gate(x)) * self.shared_expert(x) + mx.eval(y) + _tick("combine", t) + PROF["n_calls"] += 1 + return y diff --git a/mlx_streaming/core/moe/compute.py b/mlx_streaming/core/moe/compute.py new file mode 100644 index 0000000..80bd4ec --- /dev/null +++ b/mlx_streaming/core/moe/compute.py @@ -0,0 +1,208 @@ +"""MoE 专家计算：把选中专家切片成小 SwitchGLU 并前向（数值等价全量 SwitchGLU）。 + +设计要点： +- `_unique_and_local`：把 top-k 路由的全局专家 id 压成「唯一集合 + 本地下标」， + 只在实际激活的少数专家上 gather 计算。 +- `PersistentSubGLU`：跨调用复用一套 SwitchGLU + 3×QuantizedSwitchLinear 对象， + 解码 batch=1 时唯一专家数恒为 top_k，只首次构造、之后原地 update 权重，省重建开销。 +""" +from typing import Tuple + +import mlx.core as mx +import mlx.nn as nn +from mlx_lm.models.switch_layers import SwitchLinear, QuantizedSwitchLinear, SwitchGLU + +from mlx_streaming import config +from mlx_streaming.core.moe.custom_kernel import ( + _custom_qproj_enabled, _custom_fused_moe_enabled, _custom_qproj_targets, + _custom_qlinear_indexed, _custom_fused_moe_indexed) + + +def _unique_and_local(inds: mx.array) -> Tuple[mx.array, mx.array]: + """返回 (uniq, local)：uniq 是排序后的唯一全局专家 id，local 是与 inds 同形状的本地下标。""" + flat = [int(i) for i in inds.reshape(-1).tolist()] + uniq_sorted = sorted(set(flat)) + remap = {g: i for i, g in enumerate(uniq_sorted)} + local = mx.array([remap[i] for i in flat], dtype=inds.dtype).reshape(inds.shape) + uniq = mx.array(uniq_sorted, dtype=inds.dtype) + return uniq, local + + +def _slice_switch_linear(lin, uniq: mx.array): + """把一个 SwitchLinear/QuantizedSwitchLinear 沿专家维切到 uniq，返回新的小 linear。""" + n = int(uniq.shape[0]) + has_bias = "bias" in lin + if isinstance(lin, QuantizedSwitchLinear): + new = QuantizedSwitchLinear( + lin.input_dims, lin.output_dims, n, bias=has_bias, + group_size=lin.group_size, bits=lin.bits, mode=lin.mode, + ) + else: + new = SwitchLinear(lin.input_dims, lin.output_dims, n, bias=has_bias) + + sliced = {} + for name, p in lin.parameters().items(): + if isinstance(p, mx.array) and p.ndim >= 1 and p.shape[0] == lin.num_experts: + sliced[name] = p[uniq] # 第 0 维是专家维，按 uniq 取子集 + else: + sliced[name] = p + new.update(sliced) + return new + + +def streaming_switch_glu_forward(glu: SwitchGLU, x: mx.array, inds: mx.array) -> mx.array: + """只在 inds 涉及到的唯一专家上做 SwitchGLU 等价计算。""" + uniq, local = _unique_and_local(inds) + sub = SwitchGLU(glu.gate_proj.input_dims, glu.gate_proj.output_dims, int(uniq.shape[0])) + sub.gate_proj = _slice_switch_linear(glu.gate_proj, uniq) + sub.up_proj = _slice_switch_linear(glu.up_proj, uniq) + sub.down_proj = _slice_switch_linear(glu.down_proj, uniq) + sub.activation = glu.activation + return sub(x, local) + + +def _build_qsl(prefix: str, fetched: dict, in_dims: int, out_dims: int, n: int, + group_size: int, bits: int): + """用 fetched 里以 prefix.* 为键的堆叠参数构造一个 QuantizedSwitchLinear。""" + lin = QuantizedSwitchLinear(in_dims, out_dims, n, bias=False, + group_size=group_size, bits=bits, mode="affine") + sub = {name.split(".", 1)[1]: v for name, v in fetched.items() + if name.startswith(prefix + ".")} + lin.update(sub) + return lin + + +def streaming_switch_glu_forward_from_store(store, layer, x, inds, hidden, moe_inter, + group_size, bits): + """文件后端版：从 store 只取选中专家、构造小 SwitchGLU 计算（等价于全量 SwitchGLU）。 + + 无状态版本（每次新建 sub），保留给单测使用；端到端走 PersistentSubGLU。 + """ + uniq, local = _unique_and_local(inds) + fetched = store.fetch(layer, [int(i) for i in uniq.tolist()]) + n = int(uniq.shape[0]) + sub = SwitchGLU(hidden, moe_inter, n) + sub.gate_proj = _build_qsl("gate_proj", fetched, hidden, moe_inter, n, group_size, bits) + sub.up_proj = _build_qsl("up_proj", fetched, hidden, moe_inter, n, group_size, bits) + sub.down_proj = _build_qsl("down_proj", fetched, moe_inter, hidden, n, group_size, bits) + return sub(x, local) + + +def _update_qsl(lin, prefix: str, fetched: dict): + """原地更新一个 QuantizedSwitchLinear 的参数（只换数组引用，不重建对象）。""" + sub = {name.split(".", 1)[1]: v for name, v in fetched.items() + if name.startswith(prefix + ".")} + lin.update(sub) + + +class PersistentSubGLU: + """按专家数 n 缓存一套 SwitchGLU + 3×QuantizedSwitchLinear，跨调用复用。 + + 解码 batch=1 时每层唯一专家数恒为 top_k，n 不变 → 只在首次/n 变化时构造一次， + 之后每个 token 仅原地 update 三组权重，省掉每调用重建对象与初始化随机量化权重的开销。 + """ + + def __init__(self, hidden: int, moe_inter: int, group_size: int, bits: int, + proj_bits: dict | None = None, layer_idx: int | None = None, + quant_mode: str = "affine", swiglu_limit: float = 0.0): + self.hidden = hidden + self.moe_inter = moe_inter + self.group_size = group_size + self.bits = bits + self.layer_idx = layer_idx + # 混合精度：每个 proj 用各自 bit；None 时三 proj 统一用 bits。 + self.proj_bits = proj_bits or { + "gate_proj": bits, "up_proj": bits, "down_proj": bits} + # quant_mode：量化模式（affine / mxfp4）；swiglu_limit>0 时走 DeepSeek 手动 clip 路径。 + self.quant_mode = quant_mode + self.swiglu_limit = swiglu_limit + self._glu = None + self._n = None + + def _ensure(self, n: int): + if self._glu is not None and self._n == n: + return + pb = self.proj_bits + glu = SwitchGLU(self.hidden, self.moe_inter, n) + glu.gate_proj = QuantizedSwitchLinear( + self.hidden, self.moe_inter, n, bias=False, + group_size=self.group_size, bits=pb["gate_proj"], mode=self.quant_mode) + glu.up_proj = QuantizedSwitchLinear( + self.hidden, self.moe_inter, n, bias=False, + group_size=self.group_size, bits=pb["up_proj"], mode=self.quant_mode) + glu.down_proj = QuantizedSwitchLinear( + self.moe_inter, self.hidden, n, bias=False, + group_size=self.group_size, bits=pb["down_proj"], mode=self.quant_mode) + self._glu = glu + self._n = n + + def forward(self, fetched: dict, n: int, x: mx.array, local: mx.array) -> mx.array: + self._ensure(n) + _update_qsl(self._glu.gate_proj, "gate_proj", fetched) + _update_qsl(self._glu.up_proj, "up_proj", fetched) + _update_qsl(self._glu.down_proj, "down_proj", fetched) + if self.swiglu_limit > 0: + # DeepSeek 路径：手动 gate/up/clip/silu/down，不能用 SwiGLU 融合激活 + # （镜像 deepseek_v4.Experts：up 双侧 clip、gate 上侧 minimum）。 + x_exp = mx.expand_dims(x, (-2, -3)) # (..., 1, 1, D) + gate = self._glu.gate_proj(x_exp, local) + up = self._glu.up_proj(x_exp, local) + up = mx.clip(up, -self.swiglu_limit, self.swiglu_limit) + gate = mx.minimum(gate, self.swiglu_limit) + h = nn.silu(gate) * up + y = self._glu.down_proj(h, local) + return y.squeeze(-2) + max_fused_seq = config.custom_fused_moe_max_seq() + if (x.shape[1] <= max_fused_seq + and _custom_fused_moe_enabled(self.layer_idx or -1, self.proj_bits)): + return self._custom_fused_forward(x, local) + max_seq = config.custom_qproj_max_seq() + if (x.shape[1] <= max_seq + and _custom_qproj_enabled(self.layer_idx or -1, self.proj_bits["gate_proj"])): + return self._custom_gate_up_forward(x, local) + return self._glu(x, local) + + def _custom_fused_forward(self, x: mx.array, local: mx.array) -> mx.array: + """完整替换 gate/up/SwiGLU/down，保持 SwitchGLU 的 [B,S,K,H] 输出契约。""" + shape = local.shape + k = int(shape[-1]) + x_flat = mx.broadcast_to(mx.expand_dims(x, -2), x.shape[:-1] + (k, x.shape[-1])) + x_flat = x_flat.reshape(-1, self.hidden).astype(mx.float32) + idx = local.reshape(-1).astype(mx.uint32) + y = _custom_fused_moe_indexed( + x_flat, idx, + self._glu.gate_proj["weight"], self._glu.gate_proj["scales"], self._glu.gate_proj["biases"], + self._glu.up_proj["weight"], self._glu.up_proj["scales"], self._glu.up_proj["biases"], + self._glu.down_proj["weight"], self._glu.down_proj["scales"], self._glu.down_proj["biases"], + self.hidden, self.moe_inter, self.group_size, self.proj_bits["gate_proj"], + ) + return y.reshape(shape + (self.hidden,)) + + def _custom_gate_up_forward(self, x: mx.array, local: mx.array) -> mx.array: + """只替换 gate/up projection；down 仍走 MLX QuantizedSwitchLinear。""" + shape = local.shape + k = int(shape[-1]) + x_flat = mx.broadcast_to(mx.expand_dims(x, -2), x.shape[:-1] + (k, x.shape[-1])) + x_flat = x_flat.reshape(-1, self.hidden).astype(mx.float32) + idx = local.reshape(-1).astype(mx.uint32) + tile = config.custom_qproj_tile() + up = _custom_qlinear_indexed( + x_flat, idx, + self._glu.up_proj["weight"], self._glu.up_proj["scales"], self._glu.up_proj["biases"], + self.moe_inter, self.hidden, self.group_size, self.proj_bits["up_proj"], tile) + gate = _custom_qlinear_indexed( + x_flat, idx, + self._glu.gate_proj["weight"], self._glu.gate_proj["scales"], self._glu.gate_proj["biases"], + self.moe_inter, self.hidden, self.group_size, self.proj_bits["gate_proj"], tile) + up = up.reshape(shape + (1, self.moe_inter)) + gate = gate.reshape(shape + (1, self.moe_inter)) + a = self._glu.activation(up, gate) + if "down" in _custom_qproj_targets(): + a_flat = a.squeeze(-2).reshape(-1, self.moe_inter).astype(mx.float32) + y = _custom_qlinear_indexed( + a_flat, idx, + self._glu.down_proj["weight"], self._glu.down_proj["scales"], self._glu.down_proj["biases"], + self.hidden, self.moe_inter, self.group_size, self.proj_bits["down_proj"], tile) + return y.reshape(shape + (self.hidden,)) + y = self._glu.down_proj(a, local) + return y.squeeze(-2) diff --git a/mlx_streaming/core/moe/custom_kernel.py b/mlx_streaming/core/moe/custom_kernel.py new file mode 100644 index 0000000..2bc6eeb --- /dev/null +++ b/mlx_streaming/core/moe/custom_kernel.py @@ -0,0 +1,266 @@ +"""自定义 Metal 算子：indexed 量化线性 + fused MoE expert（实验性加速路径）。 + +这些 kernel 走 `mx.fast.metal_kernel`，按 `indices[pair]` 选择专家权重， +直接在 packed 量化权重上算 SwiGLU/down，避免反量化中间张量。仅在对应 +CUSTOM_* 环境开关命中且 bit 宽匹配时启用，默认关闭（不改变模型数值）。 +""" +from typing import Tuple # noqa: F401 (保留以兼容历史 import) + +import mlx.core as mx + +from mlx_streaming import config +from mlx_streaming.config import parse_layers_env as _parse_layers_env + +# kernel 编译缓存：按 (tile) / (hidden,moe_inter,...) 复用已编译 kernel。 +_CUSTOM_QKERNEL_CACHE = {} +_CUSTOM_FUSED_MOE_CACHE = {} + + +def _custom_qproj_enabled(layer_idx: int, bits: int) -> bool: + if not config.custom_qproj(): + return False + if bits != config.custom_qproj_bits(): + return False + layers = _parse_layers_env("CUSTOM_QPROJ_LAYERS") + return layers is None or int(layer_idx) in layers + + +def _custom_fused_moe_enabled(layer_idx: int, proj_bits: dict) -> bool: + if not config.custom_fused_moe(): + return False + bits = config.custom_fused_moe_bits() + if any(int(proj_bits[name]) != bits for name in ("gate_proj", "up_proj", "down_proj")): + return False + layers = _parse_layers_env("CUSTOM_FUSED_MOE_LAYERS") + return layers is None or int(layer_idx) in layers + + +def _custom_qproj_targets() -> set[str]: + spec = config.custom_qproj_targets() + return {x.strip() for x in spec.split(",") if x.strip()} + + +def _custom_qlinear_indexed(x: mx.array, indices: mx.array, weight: mx.array, + scales: mx.array, biases: mx.array, + out_dim: int, in_dim: int, group_size: int, + bits: int, tile: int = 4) -> mx.array: + """indexed custom qlinear: x[p,in] 使用 indices[p] 选择专家权重，输出 [p,out]。""" + key = tile + kernel = _CUSTOM_QKERNEL_CACHE.get(key) + if kernel is None: + source = r""" + constexpr int lanes_per_row = 256 / rows_per_group; + constexpr int block_size = rows_per_group * lanes_per_row; + uint tid = thread_position_in_threadgroup.x; + uint group_id = thread_position_in_grid.x / block_size; + uint local_row = tid / lanes_per_row; + uint lane = tid % lanes_per_row; + uint global_row = group_id * rows_per_group + local_row; + if (global_row >= pairs * out_dim) return; + uint pair = global_row / out_dim; + uint out_row = global_row % out_dim; + uint expert = indices[pair]; + constexpr uint mask = (1u << bits) - 1u; + constexpr int words_per_row = (in_dim * bits) / 32; + constexpr int groups_per_row = in_dim / group_size; + threadgroup float partial[block_size]; + float acc = 0.0f; + for (int col = int(lane); col < in_dim; col += lanes_per_row) { + int bit_offset = col * bits; + int word_idx = bit_offset / 32; + int shift = bit_offset % 32; + uint base = (expert * out_dim + out_row) * words_per_row; + uint word = weight[base + word_idx]; + uint q = (word >> shift); + if (shift + bits > 32) { + uint next_word = weight[base + word_idx + 1]; + q |= (next_word << (32 - shift)); + } + q = q & mask; + int g = col / group_size; + uint sb = (expert * out_dim + out_row) * groups_per_row + g; + float wv = float(q) * scales[sb] + biases[sb]; + acc += wv * x[pair * in_dim + col]; + } + partial[tid] = acc; + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint stride = lanes_per_row / 2; stride > 0; stride >>= 1) { + if (lane < stride) { + partial[tid] += partial[tid + stride]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + if (lane == 0) { + y[global_row] = partial[0 + (local_row * lanes_per_row)]; + } + """ + kernel = mx.fast.metal_kernel( + name=f"custom_qlinear_indexed_tile{tile}", + input_names=["x", "indices", "weight", "scales", "biases"], + output_names=["y"], + source=source, + ) + _CUSTOM_QKERNEL_CACHE[key] = kernel + pairs = int(indices.size) + grid_groups = (pairs * out_dim + tile - 1) // tile + (y,) = kernel( + inputs=[x, indices.astype(mx.uint32), weight, scales, biases], + output_shapes=[(pairs, out_dim)], + output_dtypes=[mx.float32], + grid=(grid_groups * 256, 1, 1), + threadgroup=(256, 1, 1), + template=[ + ("pairs", pairs), + ("out_dim", out_dim), + ("in_dim", in_dim), + ("group_size", group_size), + ("bits", bits), + ("rows_per_group", tile), + ], + ) + return y + + +def _custom_fused_moe_indexed(x: mx.array, indices: mx.array, + gate_w: mx.array, gate_s: mx.array, gate_b: mx.array, + up_w: mx.array, up_s: mx.array, up_b: mx.array, + down_w: mx.array, down_s: mx.array, down_b: mx.array, + hidden: int, moe_inter: int, group_size: int, + bits: int) -> mx.array: + """fused MoE expert: x[p] 用 indices[p] 选择专家，输出 down 后的 hidden。""" + lanes_per_row = config.custom_fused_moe_lanes() + block_size = config.custom_fused_moe_block() + key = (hidden, moe_inter, group_size, bits, lanes_per_row, block_size) + kernel = _CUSTOM_FUSED_MOE_CACHE.get(key) + if kernel is None: + source = r""" + uint tid = thread_position_in_threadgroup.x; + uint pair = thread_position_in_grid.x / block_size; + if (pair >= pairs) return; + uint expert = indices[pair]; + constexpr uint rows_per_step = block_size / lanes_per_row; + uint local_row = tid / lanes_per_row; + uint row_lane = tid % lanes_per_row; + constexpr uint mask = (1u << bits) - 1u; + constexpr int gu_words_per_row = (hidden * bits) / 32; + constexpr int gu_groups_per_row = hidden / group_size; + constexpr int down_words_per_row = (moe_inter * bits) / 32; + constexpr int down_groups_per_row = moe_inter / group_size; + threadgroup float act[1024]; + threadgroup float gate_part[block_size]; + threadgroup float up_part[block_size]; + + // 多个 lane 协作计算同一 row，避免每个 dot 完全串行。 + for (uint row_base = 0; row_base < moe_inter; row_base += rows_per_step) { + uint row = row_base + local_row; + float gate_acc = 0.0f; + float up_acc = 0.0f; + if (row < moe_inter) { + for (uint col = row_lane; col < hidden; col += lanes_per_row) { + int bit_offset = int(col * bits); + int word_idx = bit_offset / 32; + int shift = bit_offset % 32; + uint gu_base = (expert * moe_inter + row) * gu_words_per_row; + uint qg = gate_w[gu_base + word_idx] >> shift; + uint qu = up_w[gu_base + word_idx] >> shift; + if (shift + bits > 32) { + qg |= gate_w[gu_base + word_idx + 1] << (32 - shift); + qu |= up_w[gu_base + word_idx + 1] << (32 - shift); + } + qg &= mask; + qu &= mask; + uint g = col / group_size; + uint sb = (expert * moe_inter + row) * gu_groups_per_row + g; + float xv = x[pair * hidden + col]; + gate_acc += (float(qg) * gate_s[sb] + gate_b[sb]) * xv; + up_acc += (float(qu) * up_s[sb] + up_b[sb]) * xv; + } + } + gate_part[tid] = gate_acc; + up_part[tid] = up_acc; + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint stride = lanes_per_row / 2; stride > 0; stride >>= 1) { + if (row_lane < stride) { + gate_part[tid] += gate_part[tid + stride]; + up_part[tid] += up_part[tid + stride]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + if (row_lane == 0 && row < moe_inter) { + float gate_v = gate_part[tid]; + float up_v = up_part[tid]; + float sig = 1.0f / (1.0f + exp(-gate_v)); + act[row] = gate_v * sig * up_v; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + + // down projection 输出最终 hidden。 + for (uint row_base = 0; row_base < hidden; row_base += rows_per_step) { + uint row = row_base + local_row; + float acc = 0.0f; + if (row < hidden) { + for (uint col = row_lane; col < moe_inter; col += lanes_per_row) { + int bit_offset = int(col * bits); + int word_idx = bit_offset / 32; + int shift = bit_offset % 32; + uint base = (expert * hidden + row) * down_words_per_row; + uint q = down_w[base + word_idx] >> shift; + if (shift + bits > 32) { + q |= down_w[base + word_idx + 1] << (32 - shift); + } + q &= mask; + uint g = col / group_size; + uint sb = (expert * hidden + row) * down_groups_per_row + g; + acc += (float(q) * down_s[sb] + down_b[sb]) * act[col]; + } + } + gate_part[tid] = acc; + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint stride = lanes_per_row / 2; stride > 0; stride >>= 1) { + if (row_lane < stride) { + gate_part[tid] += gate_part[tid + stride]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + if (row_lane == 0 && row < hidden) { + y[pair * hidden + row] = gate_part[tid]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + """ + kernel = mx.fast.metal_kernel( + name=f"custom_fused_moe_h{hidden}_i{moe_inter}_b{bits}_l{lanes_per_row}", + input_names=[ + "x", "indices", + "gate_w", "gate_s", "gate_b", + "up_w", "up_s", "up_b", + "down_w", "down_s", "down_b", + ], + output_names=["y"], + source=source, + ) + _CUSTOM_FUSED_MOE_CACHE[key] = kernel + pairs = int(indices.size) + (y,) = kernel( + inputs=[ + x, indices.astype(mx.uint32), + gate_w, gate_s, gate_b, + up_w, up_s, up_b, + down_w, down_s, down_b, + ], + output_shapes=[(pairs, hidden)], + output_dtypes=[mx.float32], + grid=(pairs * block_size, 1, 1), + threadgroup=(block_size, 1, 1), + template=[ + ("pairs", pairs), + ("hidden", hidden), + ("moe_inter", moe_inter), + ("group_size", group_size), + ("bits", bits), + ("lanes_per_row", lanes_per_row), + ("block_size", block_size), + ], + ) + return y diff --git a/mlx_streaming/core/moe/gate.py b/mlx_streaming/core/moe/gate.py new file mode 100644 index 0000000..14fafdf --- /dev/null +++ b/mlx_streaming/core/moe/gate.py @@ -0,0 +1,32 @@ +"""MoE 门控与专家选择：top-k 激活数开关 + 跨层专家预测（gate 前向 + argpartition）。""" +import mlx.core as mx + +from mlx_streaming import config + + +def _effective_top_k(default_k: int) -> int: + """实验开关：降低每 token 激活专家数以测速度/质量取舍。默认不改变模型。""" + override = config.moe_topk_override() + if not override: + return default_k + return max(1, min(default_k, int(override))) + + +def _predict_layer_experts(norm, gate, top_k: int, x: mx.array, mult: int) -> "tuple[dict[int, float], int]": + """用 gate(norm(x)) 预测专家集，返回 ({expert_id: 最大 softmax 分数}, num_experts)。 + + norm/gate 必须取自**目标层**（被预测的那层），以匹配 probe 验证的 + gate_L(post_attention_layernorm_L(h)) 配置（recall_miss≈0.95）。 + """ + gates = mx.softmax(gate(norm(x)), axis=-1, precise=True) + num_experts = gates.shape[-1] + k = min(num_experts, top_k * mult) + inds = mx.argpartition(gates, kth=-k, axis=-1)[..., -k:] + vals = mx.take_along_axis(gates, inds, axis=-1) + mx.eval(inds, vals) + best: "dict[int, float]" = {} + for e, s in zip(inds.reshape(-1).tolist(), vals.reshape(-1).tolist()): + e, s = int(e), float(s) + if s > best.get(e, -1.0): + best[e] = s + return best, num_experts diff --git a/mlx_streaming/core/moe/native_moe.py b/mlx_streaming/core/moe/native_moe.py new file mode 100644 index 0000000..21a7289 --- /dev/null +++ b/mlx_streaming/core/moe/native_moe.py @@ -0,0 +1,504 @@ +"""CPython native MoE backend 的 Python 包装层。""" +import importlib +import json +import os +import time +from collections import OrderedDict +from functools import lru_cache + +import mlx.core as mx +import numpy as np + +from mlx_streaming import config + + +_EXPERT_STAGE_CACHE: "OrderedDict[tuple, tuple[mx.array, mx.array, mx.array]]" = OrderedDict() +_BUNDLE_STAGE_CACHE: "OrderedDict[tuple, tuple[mx.array, ...]]" = OrderedDict() +_SLOT_POOLS: dict[tuple, "NativeComputeSlotPool"] = {} +_STAGE_STATS = { + "expert_hits": 0, + "expert_misses": 0, + "bundle_hits": 0, + "bundle_misses": 0, + "evictions": 0, + "can_checks": 0, + "can_true": 0, + "can_false": 0, + "calls": 0, + "route_sync_s": 0.0, + "stage_s": 0.0, + "enqueue_s": 0.0, +} + + +class NativeComputeSlotPool: + """按层维护 compute-buffer 常驻 slot，hit 时复用 [cap,...] MLX arrays。""" + + def __init__(self, compute_dir: str, layer: int, hidden: int, inter: int, + group: int, bits: int, num_experts: int, cap: int): + self.compute_dir = compute_dir + self.layer = int(layer) + self.hidden = int(hidden) + self.inter = int(inter) + self.group = int(group) + self.bits = int(bits) + self.num_experts = int(num_experts) + self.cap = int(cap) + self.slot_of: "OrderedDict[int, int]" = OrderedDict() + self.expert_of: list[int | None] = [None] * self.cap + self.slot_arrays = [self._empty_slot() for _ in range(self.cap)] + self.pool_arrays: tuple[mx.array, ...] | None = None + self.dirty = True + self.hits = 0 + self.misses = 0 + self.evictions = 0 + self.rebuilds = 0 + + def _empty_slot(self) -> tuple[mx.array, ...]: + gu_words = self.hidden * self.bits // 32 + gu_groups = self.hidden // self.group + down_words = self.inter * self.bits // 32 + down_groups = self.inter // self.group + return ( + mx.zeros((self.inter, gu_words), dtype=mx.uint32), + mx.zeros((self.inter, gu_groups), dtype=mx.uint16), + mx.zeros((self.inter, gu_groups), dtype=mx.uint16), + mx.zeros((self.inter, gu_words), dtype=mx.uint32), + mx.zeros((self.inter, gu_groups), dtype=mx.uint16), + mx.zeros((self.inter, gu_groups), dtype=mx.uint16), + mx.zeros((self.hidden, down_words), dtype=mx.uint32), + mx.zeros((self.hidden, down_groups), dtype=mx.uint16), + mx.zeros((self.hidden, down_groups), dtype=mx.uint16), + ) + + def _load_slot(self, expert: int) -> tuple[mx.array, ...]: + gate = _stage_one_projection_uncached( + self.compute_dir, self.layer, "gate_proj", expert, + self.inter, self.hidden, self.group, self.bits, self.num_experts) + up = _stage_one_projection_uncached( + self.compute_dir, self.layer, "up_proj", expert, + self.inter, self.hidden, self.group, self.bits, self.num_experts) + down = _stage_one_projection_uncached( + self.compute_dir, self.layer, "down_proj", expert, + self.hidden, self.inter, self.group, self.bits, self.num_experts) + return (*gate, *up, *down) + + def _assign_slot(self, expert: int) -> int: + cached = self.slot_of.get(expert) + if cached is not None: + self.slot_of.move_to_end(expert) + self.hits += 1 + return cached + self.misses += 1 + if len(self.slot_of) < self.cap: + slot = next(i for i, e in enumerate(self.expert_of) if e is None) + else: + old_expert, slot = self.slot_of.popitem(last=False) + self.expert_of[slot] = None + self.evictions += 1 + self.slot_arrays[slot] = self._load_slot(expert) + self.expert_of[slot] = expert + self.slot_of[expert] = slot + self.dirty = True + return slot + + def _rebuild_pool_arrays(self) -> None: + self.pool_arrays = tuple( + mx.stack([slot[i] for slot in self.slot_arrays], axis=0) + for i in range(9) + ) + self.rebuilds += 1 + self.dirty = False + + def acquire(self, expert_ids: list[int]) -> tuple[list[int], tuple[mx.array, ...]]: + local = [self._assign_slot(int(e)) for e in expert_ids] + if self.pool_arrays is None or self.dirty: + self._rebuild_pool_arrays() + assert self.pool_arrays is not None + return local, self.pool_arrays + + def stats(self) -> dict: + return { + "cap": self.cap, + "entries": len(self.slot_of), + "hits": self.hits, + "misses": self.misses, + "evictions": self.evictions, + "rebuilds": self.rebuilds, + } + + +def _compute_dir() -> str: + return config.compute_buffer_dir() or os.path.join( + config.expert_dir(default=""), "compute_buffers") + + +@lru_cache(maxsize=1) +def _load_ext(): + try: + return importlib.import_module("mlx_streaming.native_moe_ext") + except Exception: + return None + + +@lru_cache(maxsize=256) +def _has_layer_buffers(compute_dir: str, layer: int) -> bool: + for proj in ("gate_proj", "up_proj", "down_proj"): + for name in ("weight", "scales", "biases"): + path = os.path.join(compute_dir, f"layer{layer:02d}.{proj}.{name}.bin") + if not os.path.exists(path): + return False + return True + + +@lru_cache(maxsize=1024) +def _buffer_index_matches(compute_dir: str, layer: int, proj: str, + out_dim: int, in_dim: int, group: int, bits: int, + num_experts: int) -> bool: + path = os.path.join(compute_dir, f"layer{layer:02d}.{proj}.index.json") + if not os.path.exists(path): + return False + try: + with open(path) as f: + index = json.load(f) + except Exception: + return False + tensors = index.get("tensors", {}) + weight_shape = tensors.get("weight", {}).get("shape_per_expert") + scale_shape = tensors.get("scales", {}).get("shape_per_expert") + bias_shape = tensors.get("biases", {}).get("shape_per_expert") + expected_weight = [out_dim, in_dim * bits // 32] + expected_grouped = [out_dim, in_dim // group] + return ( + int(index.get("num_experts", -1)) == int(num_experts) + and weight_shape == expected_weight + and scale_shape == expected_grouped + and bias_shape == expected_grouped + ) + + +@lru_cache(maxsize=512) +def _has_matching_compute_buffers(compute_dir: str, layer: int, + hidden: int, inter: int, group: int, + bits: int, num_experts: int) -> bool: + if not _has_layer_buffers(compute_dir, layer): + return False + # index.json 是防呆线：bit/group 或目录错配时宁可回退 MLX，不冒险读错 mmap shape。 + return ( + _buffer_index_matches( + compute_dir, layer, "gate_proj", inter, hidden, group, bits, num_experts) + and _buffer_index_matches( + compute_dir, layer, "up_proj", inter, hidden, group, bits, num_experts) + and _buffer_index_matches( + compute_dir, layer, "down_proj", hidden, inter, group, bits, num_experts) + ) + + +@lru_cache(maxsize=256) +def _projection_memmaps(compute_dir: str, layer: int, proj: str, + out_dim: int, in_dim: int, group: int, bits: int, + num_experts: int): + words = in_dim * bits // 32 + groups = in_dim // group + base = os.path.join(compute_dir, f"layer{layer:02d}.{proj}") + weight = np.memmap( + base + ".weight.bin", + dtype=np.uint32, + mode="r", + shape=(num_experts, out_dim, words), + ) + scales = np.memmap( + base + ".scales.bin", + dtype=np.uint16, + mode="r", + shape=(num_experts, out_dim, groups), + ) + biases = np.memmap( + base + ".biases.bin", + dtype=np.uint16, + mode="r", + shape=(num_experts, out_dim, groups), + ) + return weight, scales, biases + + +def _stage_projection(compute_dir: str, layer: int, proj: str, + expert_ids: list[int], out_dim: int, in_dim: int, + group: int, bits: int, num_experts: int): + if config.native_moe_stage_cache(): + staged = [ + _stage_one_projection( + compute_dir, layer, proj, int(e), out_dim, in_dim, group, bits, num_experts) + for e in expert_ids + ] + return tuple(mx.stack([entry[i] for entry in staged], axis=0) for i in range(3)) + return _stage_projection_uncached( + compute_dir, layer, proj, expert_ids, out_dim, in_dim, group, bits, num_experts) + + +def _stage_projection_uncached(compute_dir: str, layer: int, proj: str, + expert_ids: list[int], out_dim: int, in_dim: int, + group: int, bits: int, num_experts: int): + weight, scales, biases = _projection_memmaps( + compute_dir, layer, proj, out_dim, in_dim, group, bits, num_experts) + ids = np.asarray(expert_ids, dtype=np.int64) + # advanced indexing 已经产出 numpy 临时数组；mx.array 再接管成 MLX-managed staging arrays。 + return ( + mx.array(np.asarray(weight[ids])), + mx.array(np.asarray(scales[ids])), + mx.array(np.asarray(biases[ids])), + ) + + +def _stage_one_projection_uncached(compute_dir: str, layer: int, proj: str, expert: int, + out_dim: int, in_dim: int, group: int, bits: int, + num_experts: int): + weight, scales, biases = _projection_memmaps( + compute_dir, layer, proj, out_dim, in_dim, group, bits, num_experts) + return ( + mx.array(np.asarray(weight[int(expert)])), + mx.array(np.asarray(scales[int(expert)])), + mx.array(np.asarray(biases[int(expert)])), + ) + + +def _stage_one_projection(compute_dir: str, layer: int, proj: str, expert: int, + out_dim: int, in_dim: int, group: int, bits: int, + num_experts: int): + key = (compute_dir, int(layer), proj, int(expert), out_dim, in_dim, group, bits, num_experts) + cached = _EXPERT_STAGE_CACHE.get(key) + if cached is not None: + _EXPERT_STAGE_CACHE.move_to_end(key) + _STAGE_STATS["expert_hits"] += 1 + return cached + _STAGE_STATS["expert_misses"] += 1 + staged = _stage_one_projection_uncached( + compute_dir, layer, proj, expert, out_dim, in_dim, group, bits, num_experts) + _EXPERT_STAGE_CACHE[key] = staged + _EXPERT_STAGE_CACHE.move_to_end(key) + limit = max(0, config.native_moe_stage_cache_experts()) + while limit and len(_EXPERT_STAGE_CACHE) > limit: + _EXPERT_STAGE_CACHE.popitem(last=False) + _STAGE_STATS["evictions"] += 1 + return staged + + +def _slot_pool(compute_dir: str, layer: int, hidden: int, inter: int, group: int, + bits: int, num_experts: int) -> NativeComputeSlotPool: + cap = max(1, config.native_moe_slot_cap()) + key = (compute_dir, int(layer), hidden, inter, group, bits, num_experts, cap) + pool = _SLOT_POOLS.get(key) + if pool is None: + pool = NativeComputeSlotPool( + compute_dir, int(layer), hidden, inter, group, bits, num_experts, cap) + _SLOT_POOLS[key] = pool + return pool + + +def _stage_bundle(compute_dir: str, layer: int, expert_ids: list[int], + hidden: int, inter: int, group: int, bits: int, + num_experts: int): + key = ( + compute_dir, int(layer), tuple(int(e) for e in expert_ids), + hidden, inter, group, bits, num_experts, + ) + if config.native_moe_stage_bundle_cache(): + cached = _BUNDLE_STAGE_CACHE.get(key) + if cached is not None: + _BUNDLE_STAGE_CACHE.move_to_end(key) + _STAGE_STATS["bundle_hits"] += 1 + return cached + _STAGE_STATS["bundle_misses"] += 1 + gate = _stage_projection( + compute_dir, int(layer), "gate_proj", expert_ids, inter, hidden, group, bits, num_experts) + up = _stage_projection( + compute_dir, int(layer), "up_proj", expert_ids, inter, hidden, group, bits, num_experts) + down = _stage_projection( + compute_dir, int(layer), "down_proj", expert_ids, hidden, inter, group, bits, num_experts) + staged = (*gate, *up, *down) + if config.native_moe_stage_bundle_cache(): + _BUNDLE_STAGE_CACHE[key] = staged + _BUNDLE_STAGE_CACHE.move_to_end(key) + limit = max(0, config.native_moe_stage_cache_bundles()) + while limit and len(_BUNDLE_STAGE_CACHE) > limit: + _BUNDLE_STAGE_CACHE.popitem(last=False) + _STAGE_STATS["evictions"] += 1 + return staged + + +def prefetch_native_moe_stage(layer: int, expert_ids: list[int], + hidden: int, inter: int, group: int, bits: int, + num_experts: int) -> bool: + """提前把预测专家拷入 MLX-managed staging cache，供后续 fused native op 复用。""" + if not config.native_moe(): + return False + if not config.native_moe_stage_prefetch(): + return False + if config.native_moe_synthetic(): + return False + compute_dir = _compute_dir() + if not compute_dir or not _has_matching_compute_buffers( + compute_dir, layer, hidden, inter, group, bits, num_experts): + return False + try: + uniq = list(dict.fromkeys(int(e) for e in expert_ids)) + if not uniq: + return False + for proj, out_dim, in_dim in ( + ("gate_proj", inter, hidden), + ("up_proj", inter, hidden), + ("down_proj", hidden, inter), + ): + for e in uniq: + _stage_one_projection( + compute_dir, int(layer), proj, e, out_dim, in_dim, group, bits, num_experts) + return True + except Exception: + if config.native_moe_raise(): + raise + return False + + +def stage_cache_stats() -> dict: + slot_stats = { + "pools": len(_SLOT_POOLS), + "entries": sum(len(pool.slot_of) for pool in _SLOT_POOLS.values()), + "hits": sum(pool.hits for pool in _SLOT_POOLS.values()), + "misses": sum(pool.misses for pool in _SLOT_POOLS.values()), + "evictions": sum(pool.evictions for pool in _SLOT_POOLS.values()), + "rebuilds": sum(pool.rebuilds for pool in _SLOT_POOLS.values()), + } + return { + **_STAGE_STATS, + "route_sync_s": round(_STAGE_STATS["route_sync_s"], 6), + "stage_s": round(_STAGE_STATS["stage_s"], 6), + "enqueue_s": round(_STAGE_STATS["enqueue_s"], 6), + "expert_entries": len(_EXPERT_STAGE_CACHE), + "bundle_entries": len(_BUNDLE_STAGE_CACHE), + "slot_pool": slot_stats, + } + + +def note_route_sync(seconds: float) -> None: + _STAGE_STATS["route_sync_s"] += float(seconds) + + +def can_native_moe(layer: int, hidden: int, inter: int, group: int, bits: int, + num_experts: int) -> bool: + """只做缓存化的轻量可用性判断；失败时避免上层触发 route host 同步。""" + _STAGE_STATS["can_checks"] += 1 + if not config.native_moe(): + _STAGE_STATS["can_false"] += 1 + return False + if not config.native_moe_mlx_op(): + _STAGE_STATS["can_false"] += 1 + return False + if config.native_moe_synthetic(): + ok = _load_ext() is not None + _STAGE_STATS["can_true" if ok else "can_false"] += 1 + return ok + if (hidden * bits) % 32 != 0 or (inter * bits) % 32 != 0: + _STAGE_STATS["can_false"] += 1 + return False + compute_dir = _compute_dir() + if not compute_dir: + _STAGE_STATS["can_false"] += 1 + return False + ok = ( + _load_ext() is not None + and _has_matching_compute_buffers( + compute_dir, int(layer), hidden, inter, group, bits, num_experts) + ) + _STAGE_STATS["can_true" if ok else "can_false"] += 1 + return ok + + +def try_native_moe(layer: int, expert_ids: list[int], x: mx.array, scores: mx.array, + hidden: int, inter: int, group: int, bits: int, + num_experts: int) -> "mx.array | None": + """尝试 native fused MoE，失败返回 None 让调用方回退 MLX 路径。""" + if not config.native_moe(): + return None + if not config.native_moe_mlx_op(): + return None + if (hidden * bits) % 32 != 0 or (inter * bits) % 32 != 0: + return None + if not expert_ids: + return None + if not can_native_moe(layer, hidden, inter, group, bits, num_experts): + return None + compute_dir = _compute_dir() + ext = _load_ext() + if ext is None: + return None + synthetic = config.native_moe_synthetic() + if (not synthetic and not _has_matching_compute_buffers( + compute_dir, layer, hidden, inter, group, bits, num_experts)): + return None + try: + expert_arr = mx.array([int(e) for e in expert_ids], dtype=mx.uint32) + if synthetic: + t_enqueue = time.perf_counter() + y = ext.fused_moe( + x.astype(mx.float32), + expert_arr, + scores.astype(mx.float32), + compute_dir, + int(layer), + hidden, + inter, + group, + bits, + num_experts, + True, + ) + _STAGE_STATS["enqueue_s"] += time.perf_counter() - t_enqueue + else: + t_stage = time.perf_counter() + if config.native_moe_slot_pool(): + pool = _slot_pool( + compute_dir, int(layer), hidden, inter, group, bits, num_experts) + local_slots, pool_arrays = pool.acquire(expert_ids) + local_arr = mx.array(local_slots, dtype=mx.uint32) + ( + gate_w, gate_s, gate_b, + up_w, up_s, up_b, + down_w, down_s, down_b, + ) = pool_arrays + else: + local_arr = None + ( + gate_w, gate_s, gate_b, + up_w, up_s, up_b, + down_w, down_s, down_b, + ) = _stage_bundle( + compute_dir, int(layer), expert_ids, + hidden, inter, group, bits, num_experts) + _STAGE_STATS["stage_s"] += time.perf_counter() - t_stage + t_enqueue = time.perf_counter() + if local_arr is not None: + y = ext.fused_moe_slots( + x.astype(mx.float32), + local_arr, + scores.astype(mx.float32), + gate_w, gate_s, gate_b, + up_w, up_s, up_b, + down_w, down_s, down_b, + hidden, inter, group, bits, + ) + else: + y = ext.fused_moe_staged( + x.astype(mx.float32), + scores.astype(mx.float32), + gate_w, gate_s, gate_b, + up_w, up_s, up_b, + down_w, down_s, down_b, + hidden, inter, group, bits, + ) + _STAGE_STATS["enqueue_s"] += time.perf_counter() - t_enqueue + _STAGE_STATS["calls"] += 1 + return y + except Exception: + if config.native_moe_raise(): + raise + return None diff --git a/mlx_streaming/core/prefetch/__init__.py b/mlx_streaming/core/prefetch/__init__.py new file mode 100644 index 0000000..0e0e1ef --- /dev/null +++ b/mlx_streaming/core/prefetch/__init__.py @@ -0,0 +1 @@ +"""专家预取模块：后台物化(bg_prefetch) / native staging(native_staging) / 跨层预测预取(cross_layer) / 模型 patch(patch)。""" diff --git a/mlx_streaming/core/prefetch/bg_prefetch.py b/mlx_streaming/core/prefetch/bg_prefetch.py new file mode 100644 index 0000000..a0e8503 --- /dev/null +++ b/mlx_streaming/core/prefetch/bg_prefetch.py @@ -0,0 +1,133 @@ +"""后台专家预取器：在独立 MLX stream 上物化预测专家（私有 array），交接给主线程。 + +经 gating 测试验证可行（probe_multistream_gate / probe_multistream_handoff）： +- 后台线程 `with mx.stream(s2):` 物化 + mx.eval，可与主线程计算重叠、不崩。 +- 只物化「私有」array，绝不写主线程共享池（跨 stream 写共享张量会报 no Stream 错）。 +- 主线程只消费「已 eval」的交接 array → 跨 stream 读安全。 +""" +import queue +import threading +from collections import OrderedDict + +import mlx.core as mx + +from mlx_streaming import config + + +class BackgroundExpertPrefetcher: + def __init__(self, blob_source, window: int = 3, native: bool = False): + self._src = blob_source + self._stream = mx.new_stream(mx.default_device()) + self._q: "queue.Queue" = queue.Queue() + self._ready: "OrderedDict[tuple, dict]" = OrderedDict() + self._ready_layers: "list[int]" = [] + self._window = window + self._native = native + self._lock = threading.Lock() + self._stop = False + self.submitted = 0 + self.materialized = 0 + self.taken = 0 + # 就绪率诊断：promote 时已物化好的(ready_on_time) vs 提交了但还在飞行中(not_ready)。 + # 用于量化 attention/GDN 窗口是否够长隐藏 I/O。 + self.ready_on_time = 0 + self.not_ready = 0 + self.materialize_s = 0.0 # bg 线程物化(load+eval)累计墙钟,验证"串行加性"假设 + self._native_mat = config.native_materialize() + self._inflight: "dict[int, set]" = {} + self._t = threading.Thread(target=self._loop, daemon=True) + self._t.start() + + def submit(self, layer: int, expert_ids) -> None: + ids = [int(e) for e in expert_ids] + if not ids: + return + with self._lock: + self.submitted += len(ids) + self._inflight.setdefault(int(layer), set()).update(ids) + self._q.put((int(layer), ids)) + + def _loop(self) -> None: + while not self._stop: + try: + layer, ids = self._q.get(timeout=0.1) + except queue.Empty: + continue + try: + import time as _t + _t0 = _t.perf_counter() + with mx.stream(self._stream): + # view_bf16=False：后台不做 .view(bfloat16)（会报 no Stream(gpu,0)）； + # scales/biases 留 uint16，主线程 take 时再 view。 + # NATIVE_MATERIALIZE=1：用 C++ blob_load 把拷贝挪出 GIL（减少对主线程争用）。 + if self._native_mat: + experts = self._src.load_experts_native(layer, ids, view_bf16=False) + else: + experts = self._src.load_experts(layer, ids, view_bf16=False) + mx.eval([v for d in experts.values() for v in d.values()]) + self.materialize_s += _t.perf_counter() - _t0 + with self._lock: + pend = self._inflight.get(layer) + for e, d in experts.items(): + self._ready[(layer, int(e))] = d + self.materialized += 1 + if pend is not None: + pend.discard(int(e)) + if layer not in self._ready_layers: + self._ready_layers.append(layer) + while len(self._ready_layers) > self._window: + old = self._ready_layers.pop(0) + for k in [k for k in self._ready if k[0] == old]: + del self._ready[k] + except Exception: + # 后台失败不影响主线程：主线程会优雅回退到同步 demand 路径。 + pass + + @staticmethod + def _view_bf16(d: dict) -> dict: + """主线程消费时把 scales/biases 从 uint16 位重解释回 bfloat16。""" + out = {} + for k, v in d.items(): + out[k] = v.view(mx.bfloat16) if (k.endswith(".scales") or k.endswith(".biases")) else v + return out + + def take_ready(self, layer: int, e: int) -> "dict | None": + with self._lock: + d = self._ready.pop((int(layer), int(e)), None) + if d is None: + return None + self.taken += 1 + return self._view_bf16(d) + + def take_ready_layer(self, layer: int) -> "dict[int, dict]": + """取走某层所有就绪专家：{expert_id: {proj.tensor: mx.array}}（主线程调用）。""" + raw = {} + with self._lock: + for key in [k for k in self._ready if k[0] == int(layer)]: + raw[key[1]] = self._ready.pop(key) + self.taken += 1 + return {e: self._view_bf16(d) for e, d in raw.items()} + + def ready_count(self, layer: int) -> int: + with self._lock: + return sum(1 for k in self._ready if k[0] == int(layer)) + + def note_promote(self, layer: int, ready_now: int) -> None: + """promote_prefetched 调用：记录本次 promote 时已就绪数 vs 仍在飞行中(窗口没盖住)。""" + with self._lock: + self.ready_on_time += int(ready_now) + pend = self._inflight.get(int(layer)) + if pend: + self.not_ready += len(pend) + pend.clear() + + def stats(self) -> dict: + with self._lock: + return {"submitted": self.submitted, "materialized": self.materialized, + "taken": self.taken, "ready": len(self._ready), + "ready_on_time": self.ready_on_time, "not_ready": self.not_ready, + "materialize_s": round(self.materialize_s, 3)} + + def close(self) -> None: + self._stop = True + self._t.join(timeout=2) diff --git a/mlx_streaming/core/prefetch/cross_layer.py b/mlx_streaming/core/prefetch/cross_layer.py new file mode 100644 index 0000000..35691b3 --- /dev/null +++ b/mlx_streaming/core/prefetch/cross_layer.py @@ -0,0 +1,181 @@ +"""跨层专家预取：在 attention/GDN 之前用本层输入预测目标层 MoE 路由并提前取专家。 + +通过 monkeypatch `Qwen3NextDecoderLayer.__call__`，在真正进入 MoE 计算前用 +gate_L(post_attention_layernorm_L(h)) 预测目标层专家（对齐 recall≈0.95 的探针口径）， +把 top-(top_k*mult) 专家在当前层计算窗口内异步预取，藏住读盘/物化延迟。 + +支持多条预取后端（按环境开关择一）：STREAM_BLOB_BG（后台物化进池）、STREAM_BLOB +（blob 字节预读）、STREAM_BLOB_LOADER（blob-loader 字节预热）、native stage 预取。 +""" +import time + +import mlx.core as mx + +from mlx_streaming import config +from mlx_streaming.core.moe import native_moe +from mlx_streaming.core.moe.gate import _predict_layer_experts +from mlx_streaming.core.moe.block import FileStreamingMoeBlock + +# 是否已对 decoder layer 打过预取补丁（全局只打一次）。 +_CROSS_LAYER_PREFETCH_PATCHED = False +# 每次「整模型前向」内的全局预取预算（按层序回绕重置），避免无界预取撑爆窗口。 +_FILE_PREFETCH_REMAINING = 0 +_FILE_PREFETCH_LAST_LAYER = -1 +_STAGE_PREFETCH_REMAINING = 0 +_STAGE_PREFETCH_LAST_LAYER = -1 + + +def _cross_layer_prefetch_mult() -> int: + return config.cross_layer_mult() + + +def _submit_missing_prefetch(store, layer: int, predicted) -> "list[int]": + """只把"预测∩非常驻"的专家提交给后台预取器，避免重载常驻、撑爆 attention 窗口。 + + 返回实际提交的缺失专家列表（供测试/统计）。 + """ + bg = getattr(store, "_bg", None) + if bg is None: + return [] + resident = store.resident_experts(layer) if hasattr(store, "resident_experts") else set() + missing = [int(e) for e in predicted if int(e) not in resident] + if missing: + bg.submit(int(layer), missing) + return missing + + +def _prefetch_budget(layer_idx: int, kind: str) -> int: + """按一次层序前向重置全局预算；layer 序号回绕说明进入新 forward。""" + global _FILE_PREFETCH_REMAINING, _FILE_PREFETCH_LAST_LAYER + global _STAGE_PREFETCH_REMAINING, _STAGE_PREFETCH_LAST_LAYER + if kind == "file": + total = config.file_prefetch_global_budget() + if layer_idx <= _FILE_PREFETCH_LAST_LAYER: + _FILE_PREFETCH_REMAINING = total + _FILE_PREFETCH_LAST_LAYER = layer_idx + return max(0, _FILE_PREFETCH_REMAINING) + total = config.stage_prefetch_global_budget() + if layer_idx <= _STAGE_PREFETCH_LAST_LAYER: + _STAGE_PREFETCH_REMAINING = total + _STAGE_PREFETCH_LAST_LAYER = layer_idx + return max(0, _STAGE_PREFETCH_REMAINING) + + +def _consume_prefetch_budget(n: int, kind: str) -> None: + global _FILE_PREFETCH_REMAINING, _STAGE_PREFETCH_REMAINING + if kind == "file": + _FILE_PREFETCH_REMAINING = max(0, _FILE_PREFETCH_REMAINING - n) + else: + _STAGE_PREFETCH_REMAINING = max(0, _STAGE_PREFETCH_REMAINING - n) + + +def _prefetch_layers(kind: str) -> "set[int] | None": + env = "FILE_PREFETCH_LAYERS" if kind == "file" else "STAGE_PREFETCH_LAYERS" + return config.parse_layers_env(env) + + +def enable_cross_layer_prefetch(): + """给 Qwen3NextDecoderLayer 加跨层专家预取。 + + 在 attention/GDN 之前，用当前层输入 hidden 的 post_attention_layernorm 近似预测 + 本层 MoE 路由，并把 top-(top_k*mult) 专家预取进 resident pool。 + """ + global _CROSS_LAYER_PREFETCH_PATCHED + if _CROSS_LAYER_PREFETCH_PATCHED: + return + from mlx_lm.models.qwen3_next import Qwen3NextDecoderLayer + orig_call = Qwen3NextDecoderLayer.__call__ + + def patched_call(self, x, mask=None, cache=None): + mlp = getattr(self, "mlp", None) + if mlp is not None and getattr(getattr(mlp, "store", None), "_staging", None) is not None: + # 存本层「未归一化」decoder 输入：供 native-fused-prefetch 用目标层 norm 正确预测 + # （对齐 0.95-recall 探针；MoE forward 拿到的 x 已被本层 norm 过，norm 用错会掉点）。 + mlp._unnormed_input = x + if config.cross_layer_prefetch() and config.resident_pool_enabled(): + ahead = config.cross_layer_ahead(default=0) + target_layer = getattr(self, "_layer_idx", None) + if target_layer is not None: + target_layer += ahead + target_mlp = None + target_decoder = None + if ahead == 0 and isinstance(mlp, FileStreamingMoeBlock): + target_mlp = mlp + target_decoder = self + elif target_layer is not None: + layers = getattr(getattr(self, "_prefetch_model_ref", None), "layers", []) + if 0 <= target_layer < len(layers): + target_decoder = layers[target_layer] + target_mlp = getattr(target_decoder, "mlp", None) + if isinstance(target_mlp, FileStreamingMoeBlock): + if config.probe_predict_only(): + # 诊断：只做预测 gate 前向 + eval(同步在预测上)，不 .tolist、不预取。 + # 用于拆分 32% 税：≈S(+9%)→税是 Python 编排(native 可救)；≈B(+32%)→税是 + # gate前向被 barrier 暴露在关键路径(简单 native 派发救不了)。 + g = target_mlp.gate(target_decoder.post_attention_layernorm(x)) + kk = min(g.shape[-1], 4) + ip = mx.argpartition(g, kth=-kk, axis=-1)[..., -kk:] + mx.eval(ip) + return orig_call(self, x, mask=mask, cache=cache) + # 关键：用**目标层**的 post_attention_layernorm（匹配 probe 验证的 + # gate_L(post_norm_L(h_{L-ahead})) 配置，recall_miss≈0.95）。 + best, num_experts = _predict_layer_experts( + target_decoder.post_attention_layernorm, target_mlp.gate, + target_mlp.top_k, x, _cross_layer_prefetch_mult()) + bg = getattr(target_mlp.store, "_bg", None) + if config.stream_blob_bg() and bg is not None: + # 同层(AHEAD=0)预测 + 只预取"预测∩非常驻"，藏进 attention/GDN 窗口。 + budget = config.stream_blob_bg_budget( + default=target_mlp.top_k * _cross_layer_prefetch_mult()) + picked = [e for e, _ in sorted(best.items(), key=lambda kv: kv[1], reverse=True)][:budget] + _submit_missing_prefetch(target_mlp.store, target_mlp.layer_idx, picked) + if config.window_prof(): + target_mlp._submit_t = time.perf_counter() + return orig_call(self, x, mask=mask, cache=cache) + if config.stream_blob() and target_mlp._blob is not None: + # 全流式：后台预读下一层预测专家的字节，与当前层计算重叠。 + budget = config.stream_blob_prefetch_budget(default=target_mlp.top_k * 2) + picked = [e for e, _ in sorted(best.items(), key=lambda kv: kv[1], reverse=True)][:budget] + target_mlp._blob.prefetch_async(target_mlp.layer_idx, picked) + return orig_call(self, x, mask=mask, cache=cache) + blob_loader = getattr(target_mlp.store, "_blob_loader", None) + if config.stream_blob_loader() and blob_loader is not None: + # blob-loader 路径：后台只预读字节（与当前层计算重叠）， + # demand 时主线程从预热字节快速物化进常驻池，不在主线程同步阻塞读盘。 + budget = config.stage_prefetch_per_layer_budget(default=12) + picked = [e for e, _ in sorted(best.items(), key=lambda kv: kv[1], reverse=True)][:budget] + blob_loader.prefetch_async(target_mlp.layer_idx, picked) + return orig_call(self, x, mask=mask, cache=cache) + allowed_layers = _prefetch_layers("stage") + if allowed_layers is not None and target_mlp.layer_idx not in allowed_layers: + return orig_call(self, x, mask=mask, cache=cache) + min_score = config.stage_prefetch_min_score() + per_layer = config.stage_prefetch_per_layer_budget(default=4) + remaining = _prefetch_budget(target_mlp.layer_idx, "stage") + budget = max(0, min(per_layer, remaining)) + expert_ids = [ + e for e, s in sorted(best.items(), key=lambda kv: kv[1], reverse=True) + if s >= min_score + ][:budget] + _consume_prefetch_budget(len(expert_ids), "stage") + proj_bits = getattr(target_mlp._sub, "proj_bits", {}) + bits_set = { + int(proj_bits.get(name, target_mlp.bits)) + for name in ("gate_proj", "up_proj", "down_proj") + } + if len(bits_set) == 1 and native_moe.prefetch_native_moe_stage( + target_mlp.layer_idx, + expert_ids, + target_mlp.hidden, + target_mlp.moe_inter, + target_mlp.group_size, + bits_set.pop(), + num_experts, + ): + pass + else: + target_mlp.store.prefetch(target_mlp.layer_idx, expert_ids) + return orig_call(self, x, mask=mask, cache=cache) + + Qwen3NextDecoderLayer.__call__ = patched_call + _CROSS_LAYER_PREFETCH_PATCHED = True diff --git a/mlx_streaming/core/prefetch/native_staging.py b/mlx_streaming/core/prefetch/native_staging.py new file mode 100644 index 0000000..9e5a480 --- /dev/null +++ b/mlx_streaming/core/prefetch/native_staging.py @@ -0,0 +1,229 @@ +"""native-fused-prefetch 的 miss→hit 落地：per-layer 环形 staging buffer + 池 promote。 + +机制（预读 IO 全在 GPU 完成回调里、主线程零 pread/零 .tolist/零 np→mx.array）： +- submit(layer, inds_lazy)：取该层环里的一块 buffer，调 prefetch_into_staging → + GPU 完成回调（C++）把专家字节 pread 进这块 buffer，并按 gen 原子记录 (expert→row)。 +- promote(layer, store)：读 C++ 记录（纯锁）→ 按 gen 取回 handler 真正写过的那块 buffer → + 惰性切片直接 _place_expert 写进池槽（不 eval，避免每层 host 同步拖慢 verify 流水线）。 + +为何必须 per-layer（而非全局共享小环）：预读在 GPU 完成回调里异步发生，且 batched verify +是「整块前向、最后一次 eval」——同一前向里全部 48 层的 submit 都在飞行，其 handler 可能要到 +接近末尾才触发。若用全局小环，buffer 会在它的 handler 落地/promote 消费**之前**就被后续层的 +submit 复用、被 handler 覆盖成别层数据（gen 匹配只保证 buffer 对象、保证不了内容新鲜）→ 非确定性 +串扰损坏（实测全局环 n_mismatch=60~85）。所以每层各留独立 buffer，环大小只用于跨 token 复用。 + +内存 = 层数 × ring × budget × stride。ring 默认从 4 降到 2（STAGING_RING），在保持逐位精确的前提下 +把 staging 占用减半（48×2×budget×stride）。 +""" +import os +import time + +import mlx.core as mx + +from mlx_streaming import config +from mlx_streaming.core.profiling import note_tprof, TPROF_ON + +# 诊断门控（STG_VERIFY）：开启时 promote 记录每个 (layer, expert) 落池所用 gen，供消费侧字节校验 +# 定位损坏来自哪一代 staging。默认关，对主路径零影响（关时连 dict 写入都不发生）。 +_STG_VERIFY = os.environ.get("STG_VERIFY") == "1" + + +def _typed_seg(seg, dt, shape): + """按段 dtype 重解释：uint32 原样 / uint16→bfloat16 / uint8(mxfp4 scales) 原样。""" + if dt == "uint32": + return seg.view(mx.uint32).reshape(shape) + if dt == "uint16": + return seg.view(mx.uint16).reshape(shape).view(mx.bfloat16) + return seg.reshape(shape) + + +def route_used_subset(cand: "list[int]", route_inds: mx.array, num_experts: int) -> "set[int]": + """返回 cand 中“被 route_inds 真实路由到”的子集（GPU membership，drain ≤len(cand)）。 + + cand：预读候选专家 id（host 已知，通常 ≤budget）。 + route_inds：本层真实路由 id（GPU lazy，任意 shape，内部 reshape(-1)）。 + num_experts：专家总数（建 mask 用）。 + cand 为空 → 返回空集（不触发任何 GPU op）。 + + 仅对预读候选做成员测试：不物化全量路由，只把 ≤len(cand) 个布尔拉回 host。 + """ + if not cand: + return set() + route_mask = mx.zeros((num_experts,), dtype=mx.uint8) + route_mask[route_inds.reshape(-1)] = 1 # GPU scatter（幂等） + hit = route_mask[mx.array(cand, dtype=mx.uint32)] # GPU gather，≤len(cand) + hit_list = hit.tolist() # 唯一一次极小 drain + return {int(cand[i]) for i in range(len(cand)) if hit_list[i]} + + +class NativeStagingManager: + """per-layer 环形 staging：每层 ring 块 buffer 轮转，按 gen 严格匹配回收。 + + submit 写 ring[layer][rr]、记 (gen→buffer)；C++ handler 按 gen 原子记 (expert→row)； + promote 按 gen 取回**正是 handler 写过**的那块 buffer。只要某层的 buffer 在该层下一次（ring 个 + submit 之后）被复用前，其 handler 已落地且 promote 的惰性切片已随本前向 eval 被消费，即不串扰。 + """ + def __init__(self, blob_source, budget: int, ring: int | None = None): + self.src = blob_source # BlobExpertSource：有 dir / stride / _segs + self.budget = int(budget) + self.stride = int(blob_source.stride) + # 安全下限=2：MTP 每步 verify+replay 会对同层各 submit 一次（2 个在飞），ring=1 实测串扰损坏。 + self.ring = max(2, int(ring) if ring is not None else config.staging_ring()) + self._ring: "dict[int, list]" = {} # layer -> [mx.array]*ring + self._rr: "dict[int, int]" = {} # layer -> 下一个要写的环索引 + self._gen = 1 # 全局单调 generation + self._gen_buf: "dict[int, mx.array]" = {} # gen -> 该次 submit 写的 buffer + self.submitted = 0 + self.promoted = 0 + # 诊断用:每层最近一次 promote take 到的"已就绪(C++ pread 完成)专家集"。 + # miss_attrib 据此把 miss_A 拆成 时序(不在就绪集) / 驱逐(在就绪集但 acquire 前已不在池)。 + self.last_ready: "dict[int, set]" = {} + # 诊断用(STG_VERIFY)：记录每个 (layer, expert) 最近一次被 promote 写池时所用的 gen， + # 供消费侧字节校验定位「损坏来自哪一代 staging」。默认不影响主路径(只是写 dict)。 + self.placed_gen: "dict[tuple, int]" = {} + + def _bufs(self, layer: int) -> list: + bl = self._ring.get(layer) + if bl is None: + bl = [] + for _ in range(self.ring): + a = mx.zeros((self.budget, self.stride), dtype=mx.uint8) + mx.eval(a) + bl.append(a) + self._ring[layer] = bl + self._rr[layer] = 0 + return bl + + def submit(self, layer: int, inds_lazy: mx.array, resident: "list[int] | None" = None): + """方案B：inds_lazy 为"预测宽集合"(lazy uint32 [N]，按门控分降序，N 可远大于 budget)。 + + resident：目标层提交时刻的常驻专家快照（host 侧）。C++ 回调按 resident 过滤、再按降序 + 取前 budget 个"缺口"专家 pread 进 buffer（budget=buffer 行数）。返回 dummy（折进图触发回调）。 + """ + import mlx_streaming.native_moe_ext as _N + layer = int(layer) + bufs = self._bufs(layer) + rr = self._rr[layer] + buf = bufs[rr] + self._rr[layer] = (rr + 1) % self.ring + gen = self._gen + self._gen += 1 + self._gen_buf[gen] = buf + if len(self._gen_buf) > self.ring * 64: + for g in sorted(self._gen_buf)[:len(self._gen_buf) - self.ring * 64]: + self._gen_buf.pop(g, None) + path = f"{self.src.dir}/layer{layer:02d}.blob" + self.submitted += 1 + res = [int(e) for e in (resident or [])] + # cap=self.budget：回调最多往 buffer 写 budget 行（覆盖缺口分布，p99≈15 → budget=16 够）。 + return _N.prefetch_into_staging( + buf, inds_lazy, layer, gen, path, self.stride, res, self.budget, + config.staging_pread_parallel()) + + def promote(self, layer: int, store, used: "set[int] | None" = None, + route_inds: "mx.array | None" = None, + num_experts: "int | None" = None) -> int: + """把该层 staging 里已就绪专家写进常驻池（主线程、acquire 前）。 + + used：本层真实路由专家集合（host 已知时传入）。只 promote "预读好 ∩ used" 的专家。 + route_inds/num_experts：GPU 重映射路径专用——host 没有现成 used 时， + 用 route_used_subset 仅对预读候选做 GPU membership 现算 used（drain ≤budget）。 + 三者皆缺 → 整批写入（兜底，行为同改动前）。 + """ + import mlx_streaming.native_moe_ext as _N + layer = int(layer) + _pt0 = time.perf_counter() if TPROF_ON else 0.0 + flat = _N.prefetch_staging_take(layer) + if TPROF_ON: + note_tprof("take_s", time.perf_counter() - _pt0) + if not flat: + self.last_ready[layer] = set() # 本层无就绪:miss_A 全归"时序(pread 未完成)" + if TPROF_ON: + note_tprof("promote_s", time.perf_counter() - _pt0, count_key="promote_n") + return 0 + gen = int(flat[0]) + # 记录本层"已就绪(pread 完成)"专家集:无论 buffer 是否匹配，pread 都已完成。 + self.last_ready[layer] = {int(flat[i]) for i in range(1, len(flat), 2)} + stg = self._gen_buf.pop(gen, None) # 按 handler 记的 gen 取回**正是它写过**的那块 buffer + if stg is None: + if TPROF_ON: + note_tprof("promote_s", time.perf_counter() - _pt0, count_key="promote_n") + return 0 # buffer 已被回收/不匹配 → 跳过（不会错配） + store._resident._ensure_layer(layer) # promote 在 acquire 前，池可能还没建 + resident = (store.resident_experts(layer) + if hasattr(store, "resident_experts") else set()) + pairs = [(int(flat[i]), int(flat[i + 1])) for i in range(1, len(flat), 2)] + # GPU 重映射路径无现成 used：仅对预读候选做 GPU membership 现算（不物化全量路由）。 + if used is None and route_inds is not None and num_experts is not None: + _tr0 = time.perf_counter() if TPROF_ON else 0.0 + used = route_used_subset([e for e, _ in pairs], route_inds, int(num_experts)) + if TPROF_ON: + note_tprof("route_s", time.perf_counter() - _tr0) + # 受保护（不被驱逐）= 本层真实路由（若已知），否则退化为全部预读专家。 + protect = set(used) if used is not None else {e for e, _ in pairs} + # 惰性切片直接放入池：正确性由 gen-匹配（切到 handler 真正写的那块 buffer）+ per-layer 环 + # （该 buffer 在该层 ring 次 submit 内不被覆盖，而 pool scatter 会在本前向读池时 eval）保证。 + # 不在此 mx.eval（那是每层 host 同步，在 verify 大流水线上会显著拖慢热路径）。 + # used 已知时只保留命中真实路由的专家：假阳性不写池（省 scatter、不挤掉有用专家）。 + _tpl0 = time.perf_counter() if TPROF_ON else 0.0 + pend = [(e, self._slice(stg[row])) for e, row in pairs + if e not in resident and (used is None or e in used)] + placed = 0 + for e, d in pend: + try: + store._resident._place_expert(layer, e, d, current=protect) + except ValueError: + break + if _STG_VERIFY: + self.placed_gen[(layer, e)] = gen # 诊断（默认关）：记录该专家这次落池用的 gen + placed += 1 + if TPROF_ON: + # place_s 含惰性切片构建 + 池 scatter 入图(不含其 GPU 执行);place_experts=实际写入专家数。 + note_tprof("place_s", time.perf_counter() - _tpl0, + count_key="place_experts", count=placed) + note_tprof("promote_s", time.perf_counter() - _pt0, count_key="promote_n") + self.promoted += placed + return placed + + # ---- zero-copy 双源解码：池侧区直接散写（C++ 维护侧区 e→物理行） ---- + def submit_pool_sideregion(self, layer, pred, resident, pool_list, base_row, gen=0): + """zero-copy 预取：触发 C++ 把预读专家段散写进池侧区行（填代 gen）。pool_list 为 _segs 顺序的 per-key 池数组。 + seg_nbytes 取自 blob 段表（与 pool key 顺序一致）。spec_slots = 侧区行数（=self.budget）。""" + import mlx_streaming.native_moe_ext as _N + layer = int(layer) + seg_nbytes = [int(nb) for *_, nb in self.src._segs] # 段字节（与 pool key 顺序一致） + path = f"{self.src.dir}/layer{layer:02d}.blob" + self.submitted += 1 + res = [int(e) for e in (resident or [])] + return _N.prefetch_pool_sideregion( + pool_list, seg_nbytes, pred, layer, path, self.stride, res, + int(self.budget), int(base_row), gen=int(gen)) + + def sideregion_contents(self, layer, gen=0): + """读该层某代 C++ 侧区缓存当前内容 {expert: 物理侧区行}（纯锁，不消费）。""" + import mlx_streaming.native_moe_ext as _N + flat = _N.sideregion_contents(int(layer), int(gen)) + return {int(flat[i]): int(flat[i + 1]) for i in range(0, len(flat), 2)} + + def sideregion_kv(self, layer, gen=0): + """读该层某代侧区为 (keys uint32, vals int32) 两个 device mx.array（C++ 直接建，供快路径合并）。""" + import mlx_streaming.native_moe_ext as _N + return _N.sideregion_kv(int(layer), int(gen)) + + def sideregion_reset(self): + """清空 C++ 侧区缓存（换 prompt/重置统计时调用）。""" + import mlx_streaming.native_moe_ext as _N + _N.sideregion_reset() + + def _slice(self, row: mx.array) -> dict: + # 按段 dtype 通用分派（支持 v1 affine 与 v2 mxfp4）： + # - uint32：weight，原样 reshape； + # - uint16：affine 的 scales/biases，view(bfloat16)； + # - uint8 ：mxfp4 的 scales，保持原样，绝不 view bf16。 + out = {} + off = 0 + for proj, tensor, dt, shape, nb in self.src._segs: + seg = row[off:off + nb] + out[f"{proj}.{tensor}"] = _typed_seg(seg, dt, shape) + off += nb + return out diff --git a/mlx_streaming/core/prefetch/patch.py b/mlx_streaming/core/prefetch/patch.py new file mode 100644 index 0000000..649d3bc --- /dev/null +++ b/mlx_streaming/core/prefetch/patch.py @@ -0,0 +1,56 @@ +"""模型 patch：把原生 MoE 块替换成流式块（文件后端 / 常驻切片版），并按需挂跨层预取。""" +from mlx_streaming import config +from mlx_streaming.core.moe.block import FileStreamingMoeBlock, StreamingMoeBlock +from mlx_streaming.core.prefetch.cross_layer import enable_cross_layer_prefetch + + +def patch_model_filebacked(model, store, hidden, moe_inter, group_size, bits, + proj_bits: dict | None = None, + layer_proj_bits: dict | None = None): + """把每个 MoE 块替换为 FileStreamingMoeBlock，并丢弃常驻的堆叠 switch_mlp。 + + store：FileExpertStore（所有 MoE 层共用，按 (layer,expert) 缓存）。 + proj_bits：非空时走混合精度（逐 proj 不同 bit），专家须为对应混合重量化产出。 + layer_proj_bits：{绝对层号: {proj:bits}}，非空时逐层用各自 proj_bits（优先于 proj_bits）， + 对应 requantize_dir_layered 产出。各层 QSL 用该层 bit，与流式存盘文件一一对应。 + 返回被替换的层数。被替换后原 switch_mlp 不再被引用，惰性权重不会被物化。 + """ + patched = 0 + for i, layer in enumerate(model.layers): + layer._layer_idx = i + # 用 object.__setattr__ 存反向 model 引用,避免进 nn.Module(=dict)的子模块树: + # 否则 model→layer→model 成环,wired_limit 等 tree_reduce 遍历会无限递归(RecursionError)。 + object.__setattr__(layer, "_prefetch_model_ref", model) + mlp = getattr(layer, "mlp", None) + if mlp is not None and hasattr(mlp, "switch_mlp") and hasattr(mlp, "gate"): + pb = layer_proj_bits.get(i, proj_bits) if layer_proj_bits else proj_bits + # 捕获共享专家引用使其常驻（Qwen3-Next 有，Qwen3-MoE 无） + layer.mlp = FileStreamingMoeBlock( + gate=mlp.gate, top_k=mlp.top_k, norm_topk_prob=mlp.norm_topk_prob, + store=store, layer_idx=i, hidden=hidden, moe_inter=moe_inter, + group_size=group_size, bits=bits, proj_bits=pb, + shared_expert=getattr(mlp, "shared_expert", None), + shared_expert_gate=getattr(mlp, "shared_expert_gate", None), + ) + # 同理:不进 dict,避免 mlp→model→...→mlp 成环。供 native-fused-prefetch 取下层 gate。 + object.__setattr__(layer.mlp, "_prefetch_model_ref", model) + patched += 1 + if config.cross_layer_prefetch() or getattr(store, "_staging", None) is not None: + enable_cross_layer_prefetch() + return patched + + +def patch_model(model, store_factory=None): + """把模型里每个 MoE 块（含 switch_mlp 与 gate 的块）替换成 StreamingMoeBlock。 + + store_factory(layer_idx)->LruExpertStore，可为 None（仅做 uniq 切片、不接磁盘后端）。 + 返回被替换的层数，便于校验确实命中了 MoE 层。 + """ + patched = 0 + for i, layer in enumerate(model.layers): + mlp = getattr(layer, "mlp", None) + if mlp is not None and hasattr(mlp, "switch_mlp") and hasattr(mlp, "gate"): + store = store_factory(i) if store_factory is not None else None + layer.mlp = StreamingMoeBlock(mlp, layer_idx=i, store=store) + patched += 1 + return patched diff --git a/mlx_streaming/core/profiling.py b/mlx_streaming/core/profiling.py new file mode 100644 index 0000000..5697049 --- /dev/null +++ b/mlx_streaming/core/profiling.py @@ -0,0 +1,131 @@ +"""热路径计时与诊断埋点（从 streaming_moe 抽出，集中管理）。 + +- PROF：细粒度分段计时（STREAM_PROF=1），probe 读取。 +- WINDOW_PROF：同层 submit→promote 的 attention/GDN 窗口时间（WINDOW_PROF=1）。 +- PREDICT_RECALL_PROF：运行时预测对实际路由的覆盖率（分诊预取覆盖问题，PREDICT_RECALL_PROF=1）。 +这些 dict 为共享可变对象：各模块 import 后原地累加，外部（run_mtp_spec）读同一对象。 +""" +import time + +import mlx.core as mx + +from mlx_streaming import config + +_PROF_ON = config.stream_prof() +PROF = {"route": 0.0, "pyremap": 0.0, "fetch": 0.0, "matmul": 0.0, "combine": 0.0, + "n_calls": 0} +WINDOW_PROF = {"sum_s": 0.0, "n": 0} +PREDICT_RECALL_PROF = {"hit": 0, "routed": 0, "n": 0} +# 主动预取路径 host 墙钟探针（PREFETCH_TPROF=1）：量主线程"不可与 GPU 重叠"的 CPU 时间。 +# 注意：MLX 惰性——gate matmul / pool scatter 的 GPU 计算落在前向末尾统一 eval，本探针看不到， +# 那部分用消融吞吐口径(NATIVE_NO_SUBMIT/NATIVE_NO_PROMOTE 差值)测。本探针专测 host 段： +# - predict_s：block._native_fused_prefetch 里建预测图(gate(x)+argpartition[+predicted_set tolist]) +# - submit_s ：stg.submit(prefetch_into_staging 的 host 调用，注册 GPU 完成回调) +# - promote_s：promote 总时长，再细分： +# · take_s ：prefetch_staging_take（C++ 锁读已就绪记录） +# · route_s：route_used_subset 的成员测试 .tolist()（GPU→host 同步，drain≤budget） +# · place_s：_place_expert 循环（建惰性切片 + 池 scatter 入图，不含其 GPU 执行） +# *_n 为各段调用次数（按"层·前向"计）。关时各调用点直接跳过，零开销。 +PREFETCH_TPROF = { + "predict_s": 0.0, "predict_n": 0, + "submit_s": 0.0, "submit_n": 0, + "promote_s": 0.0, "promote_n": 0, + "take_s": 0.0, "route_s": 0.0, "place_s": 0.0, "place_experts": 0, +} +TPROF_ON = config.prefetch_tprof() + + +def note_tprof(seg: str, dt: float, *, count_key: "str | None" = None, count: int = 1) -> None: + """累加预取某段 host 墙钟。dt 为秒;可选 count_key 同步累加调用/专家计数。""" + PREFETCH_TPROF[seg] += dt + if count_key is not None: + PREFETCH_TPROF[count_key] += count + + +def tprof_reset() -> None: + for k in PREFETCH_TPROF: + PREFETCH_TPROF[k] = 0 if isinstance(PREFETCH_TPROF[k], int) else 0.0 +# miss 归因（MISS_ATTRIB=1，promote 之后 / acquire 之前统计本层真实路由的命中构成）： +# - resident_hit：acquire 前已驻留（LRU 历史 + 本次 promote 命中） +# - miss_A_predicted：miss 且"在预测集里"——预测对了但没进池（budget 丢/时序没到/被驱逐） +# - miss_B_unpredicted：miss 且"不在预测集里"——预测器召回缺口（根本没预测到） +# dec_* 为 decode/verify 热路径专用桶（seq 短，与 prefill 长 seq 分离）：原 miss_attrib +# 仅埋在 host 分支、被 prefill 样本主导，无法反映 decode 热路径的真实 A/B 构成。 +MISS_ATTRIB = {"routed": 0, "resident_hit": 0, "miss_A_predicted": 0, + "miss_B_unpredicted": 0, "n": 0, + "dec_routed": 0, "dec_resident_hit": 0, "dec_miss_A": 0, + "dec_miss_B": 0, "dec_n": 0, + # miss_A 细分（decode 桶）：时序=就绪集里没有它（pread 没完成）； + # 驱逐=就绪过（pread 完成）但 acquire 前已不在池（被驱逐/没写进）。 + "dec_miss_A_timing": 0, "dec_miss_A_evicted": 0} + + +def note_miss_attrib(uniq, predicted, resident, is_decode: bool, ready=None) -> None: + """统计本层真实路由 uniq 的命中构成，分 prefill / decode 两桶累加。 + + uniq：本层真实路由专家集合；predicted：本层预测集；resident：acquire 前驻留集； + ready：本层最近 promote take 到的"已就绪(pread 完成)"专家集（用于细分 miss_A）。 + 供 host 路径与 GPU remap 路径共用，确保两条路径口径一致。 + """ + pset = predicted or set() + rset = ready or set() + for e in uniq: + MISS_ATTRIB["routed"] += 1 + if is_decode: + MISS_ATTRIB["dec_routed"] += 1 + if e in resident: + MISS_ATTRIB["resident_hit"] += 1 + if is_decode: + MISS_ATTRIB["dec_resident_hit"] += 1 + elif e in pset: + MISS_ATTRIB["miss_A_predicted"] += 1 + if is_decode: + MISS_ATTRIB["dec_miss_A"] += 1 + # 就绪集里有它=pread 完成过却仍 miss → 驱逐；否则 pread 没完成 → 时序。 + if e in rset: + MISS_ATTRIB["dec_miss_A_evicted"] += 1 + else: + MISS_ATTRIB["dec_miss_A_timing"] += 1 + else: + MISS_ATTRIB["miss_B_unpredicted"] += 1 + if is_decode: + MISS_ATTRIB["dec_miss_B"] += 1 + MISS_ATTRIB["n"] += 1 + if is_decode: + MISS_ATTRIB["dec_n"] += 1 + + +# 并集专家数探针(UNION_PROF=1):按本次前向的 seq 长度分桶,记每层路由专家"去重并集"大小。 +# seq=K(MTP verify,如 K=3)的桶即"K 个 token 的专家并集",决定池 cap 下限;seq=1 为 decode、 +# seq=chunk 为分块 prefill。值为 {seq: [sum_union, n_layer_calls]}。默认关、零开销。 +UNION_PROF: "dict[int, list]" = {} +# 分层原始样本(UNION_PROF=1)：{seq: {layer_idx: [本层每次前向的并集大小...]}}。 +# 仅聚合 [sum,n] 无法给出 cap 下限所需的 U_max/p99/分层分布，故额外留每层每次前向的原始值。 +# 样本量极小(≤层数×token数)，仅采集期占用，退出前汇总。 +UNION_SAMPLES: "dict[int, dict[int, list]]" = {} +UNION_ON = config.union_prof() + + +def note_union(seq: int, union_count: int, layer_idx: int = -1) -> None: + seq = int(seq) + union_count = int(union_count) + e = UNION_PROF.setdefault(seq, [0, 0]) + e[0] += union_count + e[1] += 1 + if layer_idx >= 0: + UNION_SAMPLES.setdefault(seq, {}).setdefault(int(layer_idx), []).append(union_count) + + +def union_reset() -> None: + UNION_PROF.clear() + UNION_SAMPLES.clear() + + +def prof_reset(): + for k in PROF: + PROF[k] = 0.0 + + +def _tick(seg, t0): + mx.eval # noqa (占位,避免误用) + PROF[seg] += time.perf_counter() - t0 diff --git a/mlx_streaming/core/route_trace.py b/mlx_streaming/core/route_trace.py new file mode 100644 index 0000000..b4249eb --- /dev/null +++ b/mlx_streaming/core/route_trace.py @@ -0,0 +1,50 @@ +"""MoE 路由 trace 采集工具（仅 probe 使用）。 + +热路径默认完全关闭；开启后会把每次 MoE block 的 layer 和专家集合记录下来， +用于离线替换策略模拟。注：开启会触发路由 ids 的 CPU 同步，不用于正式性能测试。 +""" +import json + + +_enabled = False +_events: list[dict] = [] + + +def enable() -> None: + global _enabled, _events + _enabled = True + _events = [] + + +def disable() -> None: + global _enabled + _enabled = False + + +def record(layer: int, experts, miss=None, resident=None, resident_rank=None) -> None: + if not _enabled: + return + vals = [int(e) for e in experts] + rec = {"layer": int(layer), "experts": sorted(set(vals))} + if miss is not None: + rec["miss"] = sorted({int(e) for e in miss}) + if resident is not None: + rec["resident"] = sorted({int(e) for e in resident}) + if resident_rank is not None: + rec["resident_rank"] = [[int(e), float(v)] for e, v in resident_rank.items()] + _events.append(rec) + + +def dump_jsonl(path: str) -> int: + with open(path, "w") as f: + for rec in _events: + f.write(json.dumps(rec, ensure_ascii=False) + "\n") + return len(_events) + + +def size() -> int: + return len(_events) + + +def events() -> list[dict]: + return list(_events) diff --git a/mlx_streaming/model_builder.py b/mlx_streaming/model_builder.py new file mode 100644 index 0000000..496444c --- /dev/null +++ b/mlx_streaming/model_builder.py @@ -0,0 +1,252 @@ +"""共享装配层:把"文件后端流式加载主模型 + 取 hidden + 贪心生成"等跨入口复用的逻辑 +集中在这里,而不是藏在某个入口脚本(原 validate_mtp)里。 + +依赖关系:本模块位于 core(mem/expert_store/streaming_moe)与 mtp 之上, +是把它们粘合成可运行模型的装配层,被 cli/ 各入口与测试复用。 + +环境变量: + MODEL 主模型路径(MLX 量化) + EXPERT_DIR 拆分/重量化后的 per-expert safetensors 目录 + EXPERT_SLOTS 每层常驻池容量 + EXPERT_POOL_PROFILE 每层池预算 JSON(无损省内存,可选) + HIDDEN_VARIANT pre_final_norm(默认)| post_final_norm(排错时切换) +""" +import json +import os + +import mlx.core as mx +from mlx_lm import load +from mlx_lm.models.base import create_attention_mask, create_ssm_mask + +from mlx_streaming import config +from mlx_streaming.core.cache.expert_store import FileExpertStore +from mlx_streaming.core.prefetch.patch import patch_model_filebacked + +MODEL = config.model_path() +EXPERT_DIR = config.expert_dir() +EXPERT_SLOTS = config.expert_slots() +# pre_final_norm(默认)| post_final_norm(排错时切换) +HIDDEN_VARIANT = config.hidden_variant() + +# 默认 profile 文件名:放在 EXPERT_DIR 下随专家目录一起走,常用跑法自动启用(无损省内存) +DEFAULT_PROFILE_NAME = "pool_profile.json" + + +def load_pool_profile(expert_dir: str) -> "dict[int, int] | None": + """解析每层池预算 profile,返回 layer_caps 或 None。 + + 优先级:环境变量 EXPERT_POOL_PROFILE 显式指定路径 > {expert_dir}/pool_profile.json 默认。 + EXPERT_POOL_PROFILE=none/0/off 显式关闭(回到 uniform capacity)。 + profile 无损:仅按各层真实工作集分配,命中率/输出/吞吐不变(caps 仍被 capacity 上限钳制)。 + """ + p = config.expert_pool_profile() + if p.lower() in ("none", "0", "off"): + return None + if not p: # 未显式指定 → 默认找专家目录下的 profile + cand = os.path.join(expert_dir, DEFAULT_PROFILE_NAME) + p = cand if os.path.exists(cand) else "" + if p and os.path.exists(p): + with open(p) as f: + caps = json.load(f).get("layer_caps", {}) + return {int(k): int(v) for k, v in caps.items()} + return None + + +def build_streaming_model(): + """用文件后端流式 patch 加载主模型(32GB 机器装不下 41GB 非流式)。""" + model, tok = load(MODEL, lazy=True) + # 取首个 MoE 维度 + dims = None + for layer in model.layers: + mlp = getattr(layer, "mlp", None) + if mlp is not None and hasattr(mlp, "switch_mlp") and hasattr(mlp, "gate"): + gp = mlp.switch_mlp.gate_proj + dims = {"hidden": gp.input_dims, "moe_inter": gp.output_dims, + "group_size": getattr(gp, "group_size", 64), + "bits": getattr(gp, "bits", 4)} + break + bits, group, proj_bits, layer_proj_bits = ( + dims["bits"], dims["group_size"], None, None) + meta_path = os.path.join(EXPERT_DIR, "_split_meta.json") + if os.path.exists(meta_path): + with open(meta_path) as f: + meta = json.load(f) + ed = meta.get("dims", {}) + bits = ed.get("bits", bits) + group = ed.get("group_size", group) + proj_bits = ed.get("proj_bits") + if "per_layer_proj_bits" in ed: + layer_proj_bits = {int(k): v for k, v in ed["per_layer_proj_bits"].items()} + # 每层池预算 profile(pool_footprint 产出):默认从 {EXPERT_DIR}/pool_profile.json 自动启用, + # 无损省内存(命中率/输出/吞吐不变，仅不再为低占用层预留满 capacity)。 + layer_caps = load_pool_profile(EXPERT_DIR) + store = FileExpertStore(EXPERT_DIR, capacity=EXPERT_SLOTS, layer_caps=layer_caps) + if config.zerocopy_dual_source(): + # 零拷贝双源双缓冲：常驻池换成侧区模式（预分配 cap+2*spec_slots 行、禁 grow），复用原池 loader/cap/profile。 + from mlx_streaming.core.cache.resident_pool import ResidentExpertPool + _old = store._resident + # 默认单缓冲(持久 LFU,一份工作集,省一半侧区内存=生产路径);仅显式 legacy(SIDEREGION_LFU=0)用双缓冲。 + _spec_gens = 1 if config.sideregion_lfu() else 2 + store._resident = ResidentExpertPool( + _old.capacity, loader=_old.loader, layer_caps=_old.layer_caps, + spec_slots=config.pool_spec_slots(), + spec_gens=_spec_gens) + if config.stream_blob_loader(): + # blob 接入常驻池 miss-loader：复用 GPU-remap 快路径，小 EXPERT_SLOTS 即低内存。 + store._blob_loader = _make_blob_source(dims, group, bits) + # 主动预取（native-fused-prefetch miss→hit）：opt-in（NATIVE_FUSED_PREFETCH=1）。 + # 经"promote 只写真实路由命中专家"修正后已是净正：易缓存基座上 +15.5% tok/s + # （demand 11.86→13.70，hit 0.731→0.851，读盘 −45%；见 active-prefetch-turnaround-2026-06-17.md）。 + # 默认关只因收益依赖场景（基座可缓存性/是否磁盘受限）且只影响速度不影响质量、多占少量 staging 内存， + # 故作 opt-in 而非默认路径，落地配方见上述报告 §6。 + if config.native_fused_prefetch() and getattr(store, "_blob_loader", None) is not None: + try: + import mlx_streaming.native_moe_ext # noqa: F401 确认扩展已编译 + from mlx_streaming.core.prefetch.native_staging import NativeStagingManager + _budget = (config.pool_spec_slots() if config.zerocopy_dual_source() + else config.stream_blob_bg_budget(default=16)) + store._staging = NativeStagingManager(store._blob_loader, budget=_budget) + except Exception: + store._staging = None # 扩展不可用 → 关闭，不影响主路径 + # 零拷贝双源不变量：staging 侧区行数必须等于池 spec_slots，否则 C++ 会越界写池（静默损坏）。 + if config.zerocopy_dual_source() and getattr(store, "_staging", None) is not None: + assert store._staging.budget == store._resident.spec_slots, ( + f"零拷贝双源要求 staging.budget({store._staging.budget}) " + f"== 池 spec_slots({store._resident.spec_slots})") + if config.stream_blob_bg(): + # 后台预取池预填：bg 在独立 stream 物化预测专家，promote 写进池槽（需 CROSS_LAYER_PREFETCH=1）。 + from mlx_streaming.core.prefetch.bg_prefetch import BackgroundExpertPrefetcher + src = _make_blob_source(dims, group, bits) + store._blob_loader = src + store._bg = BackgroundExpertPrefetcher( + src, window=config.stream_blob_window()) + patch_model_filebacked(model, store, dims["hidden"], dims["moe_inter"], + group, bits, proj_bits=proj_bits, + layer_proj_bits=layer_proj_bits) + # 双源双缓冲：构造一个共享 VirtualPool（gen 跨层全局、每前向 +1）挂到每个流式 MoE 块。 + if config.zerocopy_dual_source() and getattr(store, "_staging", None) is not None: + from mlx_streaming.core.cache.virtual_pool import VirtualPool + from mlx_streaming.core.moe.block import FileStreamingMoeBlock + # 双源模式仍需 ahead 调度：block._native_fused_prefetch 靠 target_for 选目标层， + # 不传调度参数会让 target_for 恒返回 0（_num_layers=0）→ 预取全跳过、侧区永远空。 + _vpool = VirtualPool(store._resident, store._staging, config.pool_spec_slots(), + num_layers=len(model.layers), + cutoff=config.cross_layer_cutoff(), + ahead_lo=config.cross_layer_ahead_lo(), + ahead_hi=config.cross_layer_ahead_hi(), + store=store) + for layer in model.layers: + mlp = getattr(layer, "mlp", None) + if isinstance(mlp, FileStreamingMoeBlock): + mlp._vpool = _vpool + # 主动预取（非 zerocopy）：挂 per-layer ahead 调度器 vpool（cutoff），让晚层预读更早发起。 + if (config.native_fused_prefetch() and not config.zerocopy_dual_source() + and getattr(store, "_staging", None) is not None): + from mlx_streaming.core.cache.virtual_pool import VirtualPool + from mlx_streaming.core.moe.block import FileStreamingMoeBlock + _sched = VirtualPool(num_layers=len(model.layers), + cutoff=config.cross_layer_cutoff(), + ahead_lo=config.cross_layer_ahead_lo(), + ahead_hi=config.cross_layer_ahead_hi()) + for layer in model.layers: + mlp = getattr(layer, "mlp", None) + if isinstance(mlp, FileStreamingMoeBlock): + mlp._vpool = _sched + if config.stream_blob(): + _attach_blob_source(model, dims, group, bits) + # KV 量化(IsoQuant K4/V3 + SO(4) 旋转):仅作用于 12 个全注意力层,128k KV 3.0→~0.68 GiB。 + if config.kv_quant(): + from mlx_streaming.core.cache.kv_quant_patch import patch_kv_quant + patch_kv_quant(model, + group_size=config.kv_group_size(), + k_bits=config.kv_k_bits(), + v_bits=config.kv_v_bits(), + rotate=config.kv_rotate(), + seed=config.kv_rot_seed()) + # AUTOPIN 预热钉死(池 + blob_loader + staging 就绪后、return 前):按历史路由热度把每层 + # top-N 热专家预填进常驻池并 pin 住(不参与任何驱逐),消灭冷启动慢热;usage 缺失则跳过 + # (会话中计数仍在累计,落盘供下次启动)。AUTOPIN=0 零行为。 + if config.autopin(): + from mlx_streaming.core.cache import autopin as _autopin + _ap = _autopin.warm_start_pin(store) + if _ap["pinned"]: + print(f"[AUTOPIN] 预热 pin: {_ap['layers']} 层共 {_ap['pinned']} 专家 " + f"(frac={config.autopin_budget_frac()}, 耗时 {_ap['seconds']}s)" + + (f", 跳过 {_ap['skipped_layers']} 层" if _ap["skipped_layers"] else ""), + flush=True) + # 把 store 挂到 model 上，供只拿到 model 的调用方(server /api/stats 读专家池健康度)取用。 + # 非 array/dict/list/tuple 走 object 属性，不进 nn.Module 参数树。 + model._expert_store = store + return model, tok, store + + +def _make_blob_source(dims, group, bits): + from mlx_streaming.core.cache.blob_loader import BlobExpertSource + blob_dir = config.blob_dir() or os.path.join(EXPERT_DIR, "blobs") + workers = config.stream_blob_workers() + nocache = config.stream_blob_nocache(default="0") + num_experts = 512 + idx_path = os.path.join(blob_dir, "blob_index.json") + if os.path.exists(idx_path): + with open(idx_path) as f: + num_experts = int(json.load(f).get("num_experts", num_experts)) + return BlobExpertSource(blob_dir, dims["hidden"], dims["moe_inter"], group, bits, + num_experts=num_experts, workers=workers, nocache=nocache) + + +def _attach_blob_source(model, dims, group, bits): + """STREAM_BLOB=1：给每个流式 MoE 块注入共享 BlobExpertSource（全流式低内存路径）。""" + from mlx_streaming.core.cache.blob_loader import BlobExpertSource + from mlx_streaming.core.moe.block import FileStreamingMoeBlock + + blob_dir = config.blob_dir() or os.path.join(EXPERT_DIR, "blobs") + workers = config.stream_blob_workers() + nocache = config.stream_blob_nocache(default="1") + num_experts = 512 + idx_path = os.path.join(blob_dir, "blob_index.json") + if os.path.exists(idx_path): + with open(idx_path) as f: + num_experts = int(json.load(f).get("num_experts", num_experts)) + src = BlobExpertSource(blob_dir, dims["hidden"], dims["moe_inter"], group, bits, + num_experts=num_experts, workers=workers, nocache=nocache) + for layer in model.layers: + mlp = getattr(layer, "mlp", None) + if isinstance(mlp, FileStreamingMoeBlock): + mlp._blob = src + + +def capture_prenorm_hidden(model, input_ids: mx.array) -> mx.array: + """跑主模型层循环但跳过最后的 model.norm,返回 last-layer hidden(norm 前)。 + + HIDDEN_VARIANT=post_final_norm 时返回 norm 之后(用于消歧排错)。 + """ + inner = model.model + h = inner.embed_tokens(input_ids) + layers = inner.layers + if not layers: + return h + cache = model.make_cache() + fa_idx = next((i for i, l in enumerate(layers) if not l.is_linear), 0) + ssm_idx = next((i for i, l in enumerate(layers) if l.is_linear), 0) + fa_mask = create_attention_mask(h, cache[fa_idx]) + ssm_mask = create_ssm_mask(h, cache[ssm_idx]) + for layer, c in zip(layers, cache): + mask = ssm_mask if layer.is_linear else fa_mask + h = layer(h, mask=mask, cache=c) + if HIDDEN_VARIANT == "post_final_norm": + h = inner.norm(h) + return h + + +def greedy(model, input_ids: mx.array, n: int) -> mx.array: + """主模型贪心生成 n 个 token,返回拼接后的完整序列(用作自投机参考)。""" + cache = model.make_cache() + cur = input_ids + out = [] + for _ in range(n): + logits = model(cur, cache=cache) + nxt = mx.argmax(logits[:, -1, :], axis=-1, keepdims=True) + out.append(nxt) + cur = nxt + mx.eval(nxt) + return mx.concatenate([input_ids] + out, axis=1) diff --git a/mlx_streaming/models/__init__.py b/mlx_streaming/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mlx_streaming/mtp/__init__.py b/mlx_streaming/mtp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mlx_streaming/mtp/bench_verdict.py b/mlx_streaming/mtp/bench_verdict.py new file mode 100644 index 0000000..f6b918a --- /dev/null +++ b/mlx_streaming/mtp/bench_verdict.py @@ -0,0 +1,28 @@ +"""最小树 A/B 评测的纯裁决逻辑:中位数 + go/no-go 判定(无副作用,可单测)。""" + + +def median(xs): + """样本中位数(偶数个取中间两数均值)。xs 非空。""" + s = sorted(xs) + n = len(s) + mid = n // 2 + if n % 2 == 1: + return s[mid] + return (s[mid - 1] + s[mid]) / 2 + + +def verdict_from_delta(delta, exact_all, margin=0.05): + """按跨 prompt 中位相对提速 delta 与 lossless 门给出裁决。 + + - exact_all=False → "bug"(lossless 硬门一票否决) + - delta > margin → "go" + - delta < -margin → "no-go" + - 其余(含边界) → "even" + """ + if not exact_all: + return "bug" + if delta > margin: + return "go" + if delta < -margin: + return "no-go" + return "even" diff --git a/mlx_streaming/mtp/drafter.py b/mlx_streaming/mtp/drafter.py new file mode 100644 index 0000000..19ce82d --- /dev/null +++ b/mlx_streaming/mtp/drafter.py @@ -0,0 +1,188 @@ +"""真实 MTP drafter：把 Qwen3NextMTP 包成 mtp_generate 需要的 draft/sync 接口。""" +import mlx.core as mx + +from mlx_streaming.mtp.qwen3_next_mtp import mtp_step +from mlx_streaming.mtp.kv_cache import _snapshot, _restore + + +class MTPDrafter: + """把 Qwen3NextMTP 包成 mtp_generate 需要的 drafter 接口。""" + + def __init__(self, mtp, lm_head): + self.mtp = mtp + self.lm_head = lm_head + self.embed_tokens = mtp.embed_tokens + + def make_cache(self): + from mlx_lm.models import cache as kc + return [kc.KVCache()] # MTP 单层全注意力 + + def draft(self, H_last, x_ids, mtp_cache, K, topk: int = 0): + # 注意:每草稿一次 int(argmax) host 同步反而最快 —— 它让 MLX 把每步 draft 的 + # 图保持很小、及时释放;试过"全程 GPU argmax、末尾一次同步"攒大惰性图,draft + # 慢 5×、端到端 24.9→16.3(A/B 证伪,见报告)。保持逐步同步。 + # topk>0(探针用):额外返回每个位置 MTP 的 top-k 候选 id(降序),量树形展开的救回上界。 + drafts, cands = [], [] + h, cur = H_last, x_ids + for _ in range(K): + logits, mh = mtp_step(self.mtp, h, cur, self.lm_head, mtp_cache[0]) + lg = logits[0].reshape(-1) + if topk > 0: + order = [int(i) for i in mx.argsort(lg)[-topk:].tolist()][::-1] # 降序 top-k + d = order[0] + cands.append(order) + else: + d = int(mx.argmax(lg)) + drafts.append(d) + h, cur = mh, mx.array([[d]]) + if topk > 0: + return drafts, cands + return drafts + + def draft_adaptive(self, H_last, x_ids, mtp_cache, depth_max, tau): + """置信度门控动态深度:逐位贪婪抽,累计置信度 C=∏p_i 跌破 tau 即停,最多 depth_max 位。 + + 返回可变长度(1..depth_max)的草稿链。C 用每位 top-1 softmax 概率连乘;低置信提前收敛 + (本步只 verify 到当前深度,省后续位置的专家加载),高置信抽满到 depth_max。始终至少抽 1 位 + (depth=1 即退化为普通单 token 解码:verify 只喂 [x]、恒接受模型真值 1 个)。 + """ + drafts = [] + h, cur = H_last, x_ids + conf = 1.0 + for i in range(depth_max): + logits, mh = mtp_step(self.mtp, h, cur, self.lm_head, mtp_cache[0]) + lg = logits[0].reshape(-1) + d = int(mx.argmax(lg)) + drafts.append(d) + h, cur = mh, mx.array([[d]]) + if i + 1 < depth_max: # 末位无需再判是否加深 + conf *= float(mx.softmax(lg)[d]) + if conf < tau: + break + return drafts + + def draft_adaptive_tree(self, H_last, x_ids, mtp_cache, depth_max, tau): + """置信度门控变长 + pos0 分支:返回 (chainA, chainB)。 + + chainA 同 draft_adaptive(累计置信度门控,变长 n=1..depth_max);chainB 是第1 位 top-2 分支, + 续抽到与 chainA 同深度 n(仅 n>=2 时,浅步 depth=1 无位置可救 → chainB=None)。用快照隔离 + A/B 分叉,B 不带 A 的递归污染。供合并路径:动态定深 + 对深链做 pos0 救回。 + """ + logits1, mh1 = mtp_step(self.mtp, H_last, x_ids, self.lm_head, mtp_cache[0]) + lg = logits1[0].reshape(-1) + top2 = [int(i) for i in mx.argsort(lg)[-2:].tolist()][::-1] # [d1a, d1b] 降序 + d1a, d1b = top2[0], top2[1] + snap_after_x = _snapshot(mtp_cache) + + # chainA:从 d1a 续抽,累计置信度跌破 tau 即停(与 draft_adaptive 同语义)。 + chainA = [d1a] + conf = float(mx.softmax(lg)[d1a]) + h, cur = mh1, mx.array([[d1a]]) + while len(chainA) < depth_max and conf >= tau: + lo, mh = mtp_step(self.mtp, h, cur, self.lm_head, mtp_cache[0]) + lg2 = lo[0].reshape(-1) + d = int(mx.argmax(lg2)) + chainA.append(d) + conf *= float(mx.softmax(lg2)[d]) + h, cur = mh, mx.array([[d]]) + + n = len(chainA) + chainB = None + if n >= 2: # 浅步无后续位置可救,不抽 B + _restore(mtp_cache, snap_after_x) + chainB = [d1b] + h, cur = mh1, mx.array([[d1b]]) + for _ in range(n - 1): + lo, mh = mtp_step(self.mtp, h, cur, self.lm_head, mtp_cache[0]) + d = int(mx.argmax(lo[0])) + chainB.append(d) + h, cur = mh, mx.array([[d]]) + return chainA, chainB + + def draft_tree(self, H_last, x_ids, mtp_cache, K, pos1=True): + """最小树:第1(始终)、第2(pos1=True 时)草稿位置展开 top-2,返回三条链,各长 K。 + + - chainA = [d1a(top-1), d2a(top-1), d3a] 全 top-1,同 draft。 + - chainB = [d1b(top-2), d2b, d3b] 第1 草稿位置 top-2 分支 → pos0 救回。 + - chainC = [d1a, d2c(第2 位次选), d3c] 或 None 第2 草稿位置 top-2 分支 → pos1 救回。 + + 各分叉点用 mtp_cache 快照隔离,保证任一分支不带其它分支的递归污染:snap_after_x(x 处理后) + 分叉 d1a/d1b;snap_after_d1a(d1a 处理后)分叉 d2a/d2c。探针实测第2 位「首选错次选对」比例 + (~11%)高于第1 位(~7%),故 pos1 分支值得抽;但每步多一次 MTP 续抽,pos1=False 时退化为 + 仅 chainA/chainB(与旧最小树成本一致),便于 A/B 隔离 pos1 增量。 + """ + logits1, mh1 = mtp_step(self.mtp, H_last, x_ids, self.lm_head, mtp_cache[0]) + lg = logits1[0].reshape(-1) + top2 = [int(i) for i in mx.argsort(lg)[-2:].tolist()][::-1] # [d1a, d1b] 降序 + d1a, d1b = top2[0], top2[1] + snap_after_x = _snapshot(mtp_cache) # x 处理后、第1 位分叉前的 MTP 递归态 + + def _continue(first, h0, n): + """首 token first 已定,从隐状态 h0 再贪婪续抽 n 个,返回长度 1+n 的链。""" + chain = [first] + h, cur = h0, mx.array([[first]]) + for _ in range(n): + lo, mh = mtp_step(self.mtp, h, cur, self.lm_head, mtp_cache[0]) + d = int(mx.argmax(lo[0])) + chain.append(d) + h, cur = mh, mx.array([[d]]) + return chain + + chainC = None + if pos1: + # 从 d1a 续抽第2 位,捕获第2 位 top-2(d2a/d2c),供 chainA 与 chainC 分叉。 + lo2, mh_after_d1a = mtp_step(self.mtp, mh1, mx.array([[d1a]]), self.lm_head, mtp_cache[0]) + lg2 = lo2[0].reshape(-1) + top2_p1 = [int(i) for i in mx.argsort(lg2)[-2:].tolist()][::-1] # [d2a, d2c] 降序 + d2a, d2c = top2_p1[0], top2_p1[1] + snap_after_d1a = _snapshot(mtp_cache) # d1a 处理后、第2 位分叉前的递归态 + chainA = [d1a] + _continue(d2a, mh_after_d1a, K - 2) # [d1a, d2a, d3a...] + _restore(mtp_cache, snap_after_d1a) # 回到 d1a 态,C 链从第2 位次选分叉 + chainC = [d1a] + _continue(d2c, mh_after_d1a, K - 2) # [d1a, d2c, d3c...] + else: + chainA = _continue(d1a, mh1, K - 1) # 仅全 top-1 链(旧最小树成本) + + # chainB:回到 x 态,从第1 位次选 d1b 分叉续抽。 + _restore(mtp_cache, snap_after_x) + chainB = _continue(d1b, mh1, K - 1) # [d1b, d2b, d3b...] + return chainA, chainB, chainC + + def draft_paths(self, H_last, x_ids, mtp_cache, K, P): + """完整树 batch-of-paths:位置1 展开 top-P,返回 P 条链(各长 K)。 + + 位置1 取 MTP top-P 候选 [d1_0..d1_{P-1}];每个候选从 pos1 的共享 MTP 递归态 mh1 分叉, + 贪婪续抽 K-1 个 token 成一条链。用 mtp_cache 快照保证各链从同一起点分叉、互不污染。 + P=1 退化为普通链;P=2 等价 draft_tree。 + """ + logits1, mh1 = mtp_step(self.mtp, H_last, x_ids, self.lm_head, mtp_cache[0]) + lg = logits1[0].reshape(-1) + firsts = [int(i) for i in mx.argsort(lg)[-P:].tolist()][::-1] # top-P 降序 + snap_pos1 = _snapshot(mtp_cache) + + def _continue(first): + chain = [first] + h, cur = mh1, mx.array([[first]]) + for _ in range(K - 1): + lo, mh = mtp_step(self.mtp, h, cur, self.lm_head, mtp_cache[0]) + d = int(mx.argmax(lo[0])) + chain.append(d) + h, cur = mh, mx.array([[d]]) + return chain + + paths = [] + for j, f in enumerate(firsts): + if j > 0: + _restore(mtp_cache, snap_pos1) # 回到 pos1 态,从同一起点分叉 + paths.append(_continue(f)) + return paths + + def sync(self, prev_H, rH, replay_in, mtp_cache): + """用已接受 token 的真实主模型 hidden 推进 MTP KV cache。 + + MTP 在位置 i 消费 (H_i, t_{i+1}) 预测 t_{i+2};因此提交 accepted prefix + `[t_{i+1}, ..., t_{i+n}]` 时,hidden 序列应为 `[H_i, ..., H_{i+n-1}]`。 + """ + from mlx_streaming.mtp.qwen3_next_mtp import mtp_advance + h_seq = mx.concatenate([prev_H, rH[:, :-1, :]], axis=1) + H = mtp_advance(self.mtp, h_seq, replay_in, mtp_cache[0]) + mx.eval(H) diff --git a/mlx_streaming/mtp/generate.py b/mlx_streaming/mtp/generate.py new file mode 100644 index 0000000..c334057 --- /dev/null +++ b/mlx_streaming/mtp/generate.py @@ -0,0 +1,506 @@ +"""Qwen3-Next MTP 自投机贪婪解码循环。 + +见 spec/plan: docs/superpowers/{specs,plans}/2026-06-07-qwen3next-mtp-self-speculation*。 + +核心:每步主模型前向出 hidden -> MTP 自回归抽 K 草稿 -> 主模型并行验证 -> +接受最长命中前缀 -> cache 快照/恢复/重放回滚(统一处理 ArraysCache 与 KVCache), +保证输出与非投机贪婪逐 token 等价。 + +cache 校验/提交机制见 mtp/kv_cache.py；drafter 接口见 mtp/drafter.py。 +""" +import time + +import mlx.core as mx +from mlx_lm.models.base import create_attention_mask, create_ssm_mask + +from mlx_streaming import config +from mlx_streaming.mtp.kv_cache import ( + enable_qwen3next_speculative_checkpoints, begin_speculative_checkpoints, + _snapshot, _restore, commit_verified_prefix, commit_verified_snapshot, + tile_caches, commit_tree_row) + + +def _batch_direct_commit_guaranteed(model) -> bool: + """判断该模型的 batch verify 是否「必定」能直接提交(commit_verified_prefix 恒成功)。 + + commit 成功的充要条件是每个 cache 都满足:可裁剪(KVCache)或已捕获 per-token checkpoint。 + - 非线性层 → KVCache,可裁剪,必成功; + - 线性层 → 必须是被 patch 过的 Qwen3NextGatedDeltaNet,verify 前向才会写 `_spec_checkpoints`。 + 全部满足才返回 True——此时每步的 `snap_m` 回退快照永远用不上,可安全省略以压低在途峰值。 + 通用/玩具递归层(未 patch)返回 False,保留 snap_m 走安全 replay 兜底(与原行为逐 token 等价)。 + """ + import mlx_streaming.mtp.kv_cache as _kv + if not getattr(_kv, "_QWEN3NEXT_CHECKPOINTS_PATCHED", False): + return False + try: + from mlx_lm.models.qwen3_next import Qwen3NextGatedDeltaNet + layers = model.model.layers + except (ImportError, AttributeError): + return False + for l in layers: + if getattr(l, "is_linear", False) and not isinstance( + getattr(l, "linear_attn", None), Qwen3NextGatedDeltaNet): + return False + return True + + +def forward_with_hidden(model, ids, cache, compute_logits: bool = True): + """跑主模型层循环 + 最终 norm,返回 (logits(1,L,V), hidden(1,L,H))。 + + logits 恒由 final-norm 后的 H 经 lm_head 得到(保证与贪婪一致)。 + 返回给 MTP drafter 的 hidden:**默认用 final-norm 之前的 h**——这与 MTP 训练/验证 + (`capture_prenorm_hidden`)消费的输入一致;喂 norm 后的 hidden 会双重归一化、显著压低 + 草稿接受率(不影响正确性,因 verify 走主模型)。`MTP_HIDDEN=post_norm` 可切回旧行为对照。 + + compute_logits=False:跳过 lm_head(2048×151936 的大投影),只更新 cache 并返回 hidden。 + 供分块 prefill 的中间块用(中间块不需要 logits,只需 cache 因果累积)。 + """ + inner = model.model + h = inner.embed_tokens(ids) + layers = inner.layers + has_full = any(not l.is_linear for l in layers) + has_linear = any(l.is_linear for l in layers) + fa_idx = next((i for i, l in enumerate(layers) if not l.is_linear), 0) + ssm_idx = next((i for i, l in enumerate(layers) if l.is_linear), 0) + fa_mask = create_attention_mask(h, cache[fa_idx]) if has_full else None + ssm_mask = create_ssm_mask(h, cache[ssm_idx]) if has_linear else None + for layer, c in zip(layers, cache): + mask = ssm_mask if layer.is_linear else fa_mask + h = layer(h, mask=mask, cache=c) + H = inner.norm(h) + hidden = H if config.mtp_hidden() == "post_norm" else h + logits = model.lm_head(H) if compute_logits else None + return logits, hidden + + +def prefill_chunked(model, ids, cache, chunk: "int | None" = None): + """分块 prefill:把 ids 按 chunk 切片逐块喂入,增量更新 cache,返回最后一块的 (logits, hidden)。 + + 整段 prefill 一次前向的激活峰值 ∝ prompt 长度(每 MoE 层瞬时物化大量唯一专家 + 长序列激活)。 + 分块后每块只算 chunk 个 token,逐块 eval 释放上一块瞬时图,峰值压回 ∝chunk,与 decode 同稳态。 + + 数值等价:全注意力层逐块因果累积与整段完全等价;线性层(gated-delta)分块在 chunk 边界处有 + 重结合的末位浮点差异(与 decode 本身逐 token 同源,可忽略)。中间块跳过 lm_head 省大投影。 + chunk<=0 或 prompt 不超过一块时,退化为整段 prefill。 + """ + if chunk is None: + chunk = config.prefill_chunk() + L = ids.shape[1] + if chunk <= 0 or L <= chunk: + return forward_with_hidden(model, ids, cache) + last_start = ((L - 1) // chunk) * chunk + for s in range(0, last_start, chunk): + _, h = forward_with_hidden(model, ids[:, s:s + chunk], cache, compute_logits=False) + mx.eval(h) # 物化本块、写 cache、释放上一块瞬时图 + return forward_with_hidden(model, ids[:, last_start:], cache) + + +def forward_with_hidden_stepwise(model, ids, cache, capture_snapshots: bool = False): + """逐 token 走主模型解码路径,可在每个 token 后保存 cache 快照。 + + 这是 exact direct-commit 的保守路径:verify 与 baseline 都走 N=1 cache 更新, + 接受前缀时直接恢复到对应 token 后的快照,不再 replay accepted prefix。 + """ + logits_parts, hidden_parts, snaps = [], [], [] + for i in range(ids.shape[1]): + logits, H = forward_with_hidden(model, ids[:, i:i + 1], cache) + mx.eval(logits, H) + logits_parts.append(logits) + hidden_parts.append(H) + if capture_snapshots and i < ids.shape[1] - 1: + snaps.append(_snapshot(cache)) + logits = mx.concatenate(logits_parts, axis=1) + H = mx.concatenate(hidden_parts, axis=1) + if capture_snapshots: + return logits, H, snaps + return logits, H + + +# ----------------------------------------------------------------- 接受判定 +def accept_prefix(drafts, preds): + """drafts: MTP 抽的 K 个草稿; preds: 主模型对应位置真实下一 token(len==K)。 + + 返回命中草稿数 matched(最长使 preds[j]==drafts[j] 的前缀长度)。 + 发射 token 由调用方组装为 drafts[:matched] + [重放末位 bonus],统一处理纠正/全命中, + 保证贪婪下逐 token 与非投机等价。 + """ + matched = 0 + for d, p in zip(drafts, preds): + if int(p) == int(d): + matched += 1 + else: + break + return matched + + +# --------------------------------------------------------- 完整树形验证(batch-of-paths) +def tree_verify_step(model, drafter, x, H_last, x_ids, mtp_cache, main_cache, K, P, snap_d): + """一次 batched 前向并行验证 P 条候选路径,提交接受最长的赢家路径。 + + 返回 (new_tokens, x_new, rH, accepted_in, matched)。 + + 原理:每条路径 `[x, d_1..d_{K-1}]` 是普通线性序列,拍到 batch 维后线性层/全注意力层都走成熟 + batch 前向(逐 row 与 batch=1 等价,见 benchmarks/test_layerwise.py)。batch=P 的计算量加宽了 + 每层预取窗口(hit_rate),多路径又提升接受长度(accept_len)。验证后按赢家 row 提取 checkpoint + /trim 提交回 batch=1 主 cache,后续解码逐 token 从赢家路径续,与非投机贪婪等价。 + """ + paths = drafter.draft_paths(H_last, x_ids, mtp_cache, K, P) + verify_in = mx.array([[x] + p[: K - 1] for p in paths]) # (P, K) + + snap_m = _snapshot(main_cache) # tile 前的 batch=1 快照(回退用) + tile_caches(main_cache, len(paths)) + begin_speculative_checkpoints(main_cache) + vlogits, vH = forward_with_hidden(model, verify_in, main_cache) + mx.eval(vlogits, vH) + + best_w, matched, preds = 0, -1, None + for wi, p in enumerate(paths): + preds_i = [int(t) for t in mx.argmax(vlogits[wi], axis=-1)] + m = accept_prefix(p, preds_i) + if m > matched: # 平局取更靠前的路径(top-1 优先) + best_w, matched, preds = wi, m, preds_i + drafts = paths[best_w] + accepted_len = min(matched + 1, K) + + committed = commit_tree_row(main_cache, verified_len=K, + accepted_len=accepted_len, row=best_w) + if snap_d is not None: + _restore(mtp_cache, snap_d) + + if not committed: + # 理论上 checkpoint 齐全不会走到;稳妥回退:恢复 batch=1 主 cache 后重放赢家 accepted prefix。 + _restore(main_cache, snap_m) + accepted_in = mx.array([[x] + drafts[:matched]]) + rlogits, rH = forward_with_hidden(model, accepted_in, main_cache) + mx.eval(rlogits) + bonus = int(mx.argmax(rlogits[:, -1, :])) + return drafts[:matched] + [bonus], bonus, rH, accepted_in, matched + + rH = vH[best_w:best_w + 1, :accepted_len, :] + accepted_in = verify_in[best_w:best_w + 1, :accepted_len] + if matched == K: + return drafts[:K], drafts[-1], rH, accepted_in, matched + return drafts[:matched] + [preds[matched]], preds[matched], rH, accepted_in, matched + + +# ----------------------------------------------------------------- 主循环 +def mtp_generate(model, drafter, tok, prompt, max_tokens, K=3, ids_mode=False, + profile=False, on_tokens=None, main_cache=None, cached_len=0): + """贪婪 MTP 自投机。 + + drafter 需提供 draft(H_last(1,1,H), x_ids(1,1), mtp_cache, K) -> list[int](长度 K); + 可选 make_cache()->list 与 sync(rH, replay_in, mtp_cache)。 + ids_mode=True 时 prompt 已是 ids(1,L) 且返回 token id 列表(测试用)。 + on_tokens:可选流式钩子,每步用「本步真正写入 produced 的 token id 列表」回调一次 + (prefill 的首 token 作为第一次回调);返回 True 表示请求尽快停止生成。 + + 跨轮复用(main_cache/cached_len):传入已持有前 cached_len 个 prompt token 的 main_cache, + 则只 prefill `prompt[:, cached_len:]`(等价于一次多 token decode,复用历史 KV/递归态,不重算)。 + 调用方须保证 prompt[:cached_len] 与 cache 中已有 token 完全一致(否则结果错误)。 + 不变式:返回时 main_cache 恰好持有 `prompt + produced[:-1]`(produced[-1] 为 pending,未入 cache), + 调用方可据此拼下轮的 cached prefix。main_cache=None 时内部新建(默认,整段 prefill)。 + """ + enable_qwen3next_speculative_checkpoints() + if main_cache is None: + main_cache = model.make_cache() + cached_len = 0 + mtp_cache = drafter.make_cache() if hasattr(drafter, "make_cache") else None + ids = prompt if ids_mode else mx.array([tok.encode(prompt)]) + if cached_len < 0 or cached_len >= ids.shape[1]: + raise ValueError( + f"cached_len={cached_len} 必须落在 [0, prompt_len={ids.shape[1]}) 内,至少留 1 个 token 供 prefill") + + # prefill(分块):只喂尚未在 cache 中的后缀 ids[:, cached_len:](cached_len==0 即整段); + # 得到第 1 个 pending token x 与其 hidden;分块把激活峰值压到与 decode 同稳态。 + logits, H = prefill_chunked(model, ids[:, cached_len:], main_cache) + x = int(mx.argmax(logits[:, -1, :])) + H_last = H[:, -1:, :] + produced = [x] + _stop_from_initial = on_tokens is not None and on_tokens([x]) + mx.eval(H_last) + t0 = time.perf_counter() + n_steps = 0 + # profile 分段累加(秒):把 sync/finalize 拆开,避免隐藏在 replay 名下。 + t_draft = t_verify = t_replay = t_snap = 0.0 + t_commit = t_sync = t_finalize = 0.0 + direct_commits = fallback_replays = replayed_tokens = 0 + # matched 直方图:accept_hist[j] = 恰好命中 j 个草稿(j∈0..K)的步数。用于实测每位置接受率, + # 不做几何分布假设。emitted/step = min(matched+1,K)，故上限为 K(本实现 verify 只喂 K-1 草稿)。 + accept_hist = [0] * (max(K, config.depth_max() if config.adaptive_depth() else K) + 1) + # top-k 覆盖探针:每位置统计模型真实 token 落在 MTP top-1/2/3 的步数(树形救回上界)。 + topk_probe = config.accept_topk() if profile else 0 + tk_n = [0] * K + tk_cover1 = [0] * K + tk_cover2 = [0] * K + tk_cover3 = [0] * K + verify_mode = config.mtp_verify_mode() + tree_mode = config.tree_top2() + tree_p1_mode = tree_mode and config.tree_top2_p1() # 第2 位(pos1)救回,附加在 tree_top2 上 + tree_verify_mode = config.tree_verify() # 完整树形验证(batch-of-paths),优先级最高 + # 置信度门控动态深度:仅在 plain 单链路径生效(与 tree/tree_verify/step 互斥)。 + adaptive_mode = config.adaptive_depth() and not tree_mode and not tree_verify_mode \ + and verify_mode != "step" + adaptive_rescue_mode = adaptive_mode and config.adaptive_rescue() # 深链叠加 pos0 救回 + conf_tau_v = config.conf_tau() + depth_max_v = max(1, config.depth_max()) + tree_P = max(1, config.tree_branches()) + tree_rescues = 0 # 第1 位 top-2 成功救回(B 链首命中)的步数 + tree_rescues_p1 = 0 # 第2 位 top-2 成功救回(C 链第2 token 命中)的步数 + # 纯 batch 直接提交路径 + 模型保证 commit 恒成功时,跳过每步一次「全 cache 深拷贝 + eval」 + # (snap_m 只用于 tree 救回 / step / replay 回退,这些路径都不满足下方条件)。省 ~72MiB 在途 + # 峰值和一次同步栅栏,数值完全不变(bit-exact,只改内存调度)。模型结构恒定,循环外算一次即可。 + _skip_snap = (verify_mode != "step") and (not tree_mode) and (not tree_verify_mode) \ + and (not adaptive_rescue_mode) and _batch_direct_commit_guaranteed(model) + + while len(produced) < max_tokens and not _stop_from_initial: + x0 = x + x_ids = mx.array([[x]]) + + snap_d = _snapshot(mtp_cache) if mtp_cache else None + prev_H_last = H_last + + # 完整树形验证:一次 batched 前向验证 P 条路径,提交赢家。自带 step 尾处理后 continue。 + if tree_verify_mode and verify_mode != "step": + if profile: + _tic = time.perf_counter() + new_tokens, x, rH, accepted_in, matched = tree_verify_step( + model, drafter, x, H_last, x_ids, mtp_cache, main_cache, K, tree_P, snap_d) + accept_hist[matched] += 1 + direct_commits += 1 + if profile: + t_verify += time.perf_counter() - _tic + _tic = time.perf_counter() + _stop = False + _n_before = len(produced) + for t in new_tokens: + produced.append(t) + if len(produced) >= max_tokens: + break + # 只把「本步真正写入 produced」的 token 交给回调:多 token 步命中 max_tokens + # 上限时,截断掉的尾巴不应上报给流式消费者(避免超报被丢弃的 token)。 + if on_tokens is not None and on_tokens(produced[_n_before:]): + _stop = True + H_last = rH[:, -1:, :] + if mtp_cache is not None and hasattr(drafter, "sync"): + drafter.sync(prev_H_last, rH, accepted_in, mtp_cache) + if profile: + t_sync += time.perf_counter() - _tic + n_steps += 1 + mx.eval(x, H_last) + if _stop: + break + continue + + if profile: + _tic = time.perf_counter() + tree_b = None + tree_c = None + step_k = K # 本步验证深度(动态深度时可变) + if tree_mode and verify_mode != "step": + # chainA(全 top-1)、chainB(第1 位 top-2 → pos0 救回)、chainC(第2 位 top-2 → pos1 救回) + drafts, tree_b, tree_c = drafter.draft_tree( + H_last, x_ids, mtp_cache, K, pos1=tree_p1_mode) + draft_cands = None + elif adaptive_mode: + # 置信度门控:返回可变长度(1..depth_max)草稿链,本步深度随置信度自适应。 + if adaptive_rescue_mode: + # 合并路径:变长 chainA + pos0 分支 chainB(深链才有),供下方 pos0 救回。 + drafts, tree_b = drafter.draft_adaptive_tree( + H_last, x_ids, mtp_cache, depth_max_v, conf_tau_v) + else: + drafts = drafter.draft_adaptive(H_last, x_ids, mtp_cache, depth_max_v, conf_tau_v) + draft_cands = None + step_k = len(drafts) + elif topk_probe > 0: + drafts, draft_cands = drafter.draft(H_last, x_ids, mtp_cache, K, topk=topk_probe) + else: + drafts, draft_cands = drafter.draft(H_last, x_ids, mtp_cache, K), None # 长度 K + verify_in = mx.array([[x] + drafts[: step_k - 1]]) # [x, d_1..d_{step_k-1}] + if profile: + mx.eval(verify_in) + t_draft += time.perf_counter() - _tic + _tic = time.perf_counter() + + snap_m = None if _skip_snap else _snapshot(main_cache) + if profile: + t_snap += time.perf_counter() - _tic + _tic = time.perf_counter() + + verify_snaps = None + if verify_mode == "step": + vlogits, vH, verify_snaps = forward_with_hidden_stepwise( + model, verify_in, main_cache, capture_snapshots=True) + else: + begin_speculative_checkpoints(main_cache) + vlogits, vH = forward_with_hidden(model, verify_in, main_cache) + mx.eval(vlogits, vH) + preds = [int(t) for t in mx.argmax(vlogits[0], axis=-1)] # 长度 K + matched = accept_prefix(drafts, preds) + # 最小树救回:A 链首草稿被拒(matched==0)且 B 候选=模型真实 token(=preds[0])时, + # 恢复 cache、改验 B 链。preds[0] 只依赖 [x],两链一致;故 d1b==preds[0] 即 B 首命中。 + # lossless(2026-07-05 定论,见 reports/tree-top2-rescue-2026-07-05.md):曾疑此救回非 + # bit-lossless,实为 demand_dual 的一个 bug——inds(argpartition 切片)是非连续视图,C++ + # 按连续读 data() 致 seq≥2 的 token≥1 装错专家,spec/救回全体失真。修 demand_dual 用 + # contiguous() 物化后,seq=K verify 与 seq=1 解码逐位等价,spec 与本救回均 bit-lossless。 + if tree_b is not None and matched == 0 and tree_b[0] == preds[0]: + _restore(main_cache, snap_m) + begin_speculative_checkpoints(main_cache) + # 重建 verify_in 为 B 链:后续 accepted_in=verify_in[:, :accepted_len] 才会取到 chainB, + # 否则残留 chainA 的错误首草稿会经 sync 污染 MTP cache、拉低后续草稿质量(低估救回收益)。 + verify_in = mx.array([[x] + tree_b[:step_k - 1]]) + vlogits, vH = forward_with_hidden(model, verify_in, main_cache) + mx.eval(vlogits, vH) + preds = [int(t) for t in mx.argmax(vlogits[0], axis=-1)] + drafts = tree_b + matched = accept_prefix(drafts, preds) + tree_rescues += 1 + # 第2 位(pos1)救回:A 链首草稿命中(matched==1)但第2 草稿被拒,且 chainC 的第2 token + # =模型真值 preds[1] 时,改验 chainC=[x, d1a, d2c, ...]。preds[1] 只依赖 [x, d1a],A/C + # 前缀一致故 preds[0]/preds[1] 不变、重验后 matched≥2;d3c 可能续命中把接受长度顶到 3。 + # 与 pos0 救回同构(纯重验、只提交模型 argmax token → bit-lossless,并集不放大)。 + elif tree_c is not None and matched == 1 and len(tree_c) > 1 and tree_c[1] == preds[1]: + _restore(main_cache, snap_m) + begin_speculative_checkpoints(main_cache) + verify_in = mx.array([[x] + tree_c[:step_k - 1]]) + vlogits, vH = forward_with_hidden(model, verify_in, main_cache) + mx.eval(vlogits, vH) + preds = [int(t) for t in mx.argmax(vlogits[0], axis=-1)] + drafts = tree_c + matched = accept_prefix(drafts, preds) + tree_rescues_p1 += 1 + accept_hist[matched] += 1 + if draft_cands is not None: + # preds[i]=模型在位置 i 的真实下一 token;查它是否在 MTP 该位置 top-1/2/3 候选里。 + for i in range(K): + p = preds[i] + c = draft_cands[i] + tk_n[i] += 1 + if p == c[0]: + tk_cover1[i] += 1 + if p in c[:2]: + tk_cover2[i] += 1 + if p in c[:3]: + tk_cover3[i] += 1 + if profile: + t_verify += time.perf_counter() - _tic + _tic = time.perf_counter() + + accepted_len = min(matched + 1, step_k) + + if verify_snaps is not None: + # step 模式：逐 token 解码路径产生的快照可精确 direct commit。 + committed = commit_verified_snapshot(main_cache, verify_snaps, + accepted_len, verified_len=K) + else: + # batch 模式：一次并行验证后按接受长度直接提交，零 replay。 + # - 可裁剪 cache（KVCache）：trim 掉 rejected 后缀即精确。 + # - 递归状态 cache（Qwen3-Next gated-delta 的 conv/ssm）：verify 前向里用 + # multistate kernel 捕获的 per-token checkpoint 与 baseline kernel 逐 bit 等价 + # （见 mtp/kv_cache.py 与 tests/test_gated_delta_multistate.py），可精确直提交。 + # commit_verified_prefix 一次处理混合 cache：KV 走 trim、Arrays 走 checkpoint 直提交。 + # 若任一递归 cache 缺 checkpoint（异常情况）则返回 False，自动回退到下方 replay。 + committed = commit_verified_prefix(main_cache, verified_len=step_k, + accepted_len=accepted_len) + if snap_d is not None: + _restore(mtp_cache, snap_d) + + accepted_in = verify_in[:, :accepted_len] + if profile: + t_commit += time.perf_counter() - _tic + _tic = time.perf_counter() + if committed: + direct_commits += 1 + # 直接使用验证前向产生的 hidden 同步 MTP cache,避免主模型 replay。 + rH = vH[:, :accepted_len, :] + if matched == step_k: + new_tokens = drafts[:step_k] + x = drafts[-1] + else: + new_tokens = drafts[:matched] + [preds[matched]] + x = preds[matched] + else: + fallback_replays += 1 + if snap_m is None: + # 理论不可达:batch 直接提交路径 checkpoint 齐全,commit 必成功。若真走到这里, + # 说明前提被破坏(如未捕获 checkpoint),此时无快照可回退,直接抛错定位而非静默错算。 + raise RuntimeError("batch verify commit failed but snapshot was skipped") + # fallback:回滚到验证前,重放 accepted prefix,保证 ArraysCache 正确。 + _restore(main_cache, snap_m) + accepted_in = mx.array([[x0] + drafts[:matched]]) + replayed_tokens += accepted_in.shape[1] + rlogits, rH = forward_with_hidden(model, accepted_in, main_cache) + mx.eval(rlogits) + bonus = int(mx.argmax(rlogits[:, -1, :])) + new_tokens = drafts[:matched] + [bonus] + x = bonus + if profile: + t_replay += time.perf_counter() - _tic + _tic = time.perf_counter() + + _n_before = len(produced) + for t in new_tokens: + produced.append(t) + if len(produced) >= max_tokens: + break + # 只把「本步真正写入 produced」的 token 交给回调:多 token 步命中 max_tokens + # 上限时,截断掉的尾巴不应上报给流式消费者(避免超报被丢弃的 token)。 + _stop = on_tokens is not None and on_tokens(produced[_n_before:]) + + H_last = rH[:, -1:, :] + if mtp_cache is not None and hasattr(drafter, "sync"): + drafter.sync(prev_H_last, rH, accepted_in, mtp_cache) + if profile: + t_sync += time.perf_counter() - _tic + _tic = time.perf_counter() + n_steps += 1 + mx.eval(x, H_last) + if profile: + t_finalize += time.perf_counter() - _tic + if _stop: + break + + produced = produced[:max_tokens] + wall = time.perf_counter() - t0 + # cache 实际驻留 token 数(以可裁剪 cache 的 offset 为准),供跨轮复用精确对账: + # 末步多 token 跨 max_tokens 会 over-commit(cache 领先于 produced),据此识别并禁用复用。 + resident_tokens = None + for c in main_cache: + if getattr(c, "is_trimmable", None) and c.is_trimmable() and hasattr(c, "offset"): + resident_tokens = int(c.offset) + break + stats = { + "steps": n_steps, + "tokens": len(produced), + "resident_tokens": resident_tokens, + "avg_accept_len": round(len(produced) / max(n_steps, 1), 3), + "wall_s": round(wall, 3), + "verify_mode": verify_mode, + "direct_commits": direct_commits, + "fallback_replays": fallback_replays, + "replayed_tokens": replayed_tokens, + "accept_hist": accept_hist, # [恰好命中0,1,...,K 个草稿的步数] + "tree_rescues": tree_rescues, # 最小树第1 位成功救回步数(tree_top2 开时) + "tree_rescues_p1": tree_rescues_p1, # 最小树第2 位成功救回步数(tree_top2 开时) + } + if topk_probe > 0: + stats["topk_probe"] = { + "topk": topk_probe, "n": tk_n, + "cover_top1": tk_cover1, "cover_top2": tk_cover2, "cover_top3": tk_cover3, + } + if profile: + seg = t_draft + t_snap + t_verify + t_commit + t_replay + t_sync + t_finalize + stats.update({ + "t_draft_s": round(t_draft, 3), + "t_snap_s": round(t_snap, 3), + "t_verify_s": round(t_verify, 3), + "t_commit_s": round(t_commit, 3), + "t_replay_s": round(t_replay, 3), + "t_sync_s": round(t_sync, 3), + "t_finalize_s": round(t_finalize, 3), + # 「重放免费」上限:草稿+验证(不含快照,因 per-token 检查点设计无需全 cache 快照) + "proj_no_replay_tps": round(len(produced) / max(t_draft + t_verify, 1e-6), 2), + "measured_seg_tps": round(len(produced) / max(seg, 1e-6), 2), + }) + if ids_mode: + return produced, stats + return tok.decode(produced), stats diff --git a/mlx_streaming/mtp/kv_cache.py b/mlx_streaming/mtp/kv_cache.py new file mode 100644 index 0000000..f1c2aa4 --- /dev/null +++ b/mlx_streaming/mtp/kv_cache.py @@ -0,0 +1,276 @@ +"""MTP 投机解码的 KV/递归状态 cache 校验机制：per-token checkpoint + 快照/恢复/提交。 + +speculative decoding 需要在「验证 K 个草稿」后只保留 accepted_len 个 token 对 cache 的 +贡献。两类 cache 处理方式不同： +- 可裁剪 cache（KVCache）：直接 trim 掉 rejected 后缀。 +- 递归状态 cache（Qwen3-Next 线性注意力 ArraysCache）：必须在 verify 前向中逐 token 记录 + `[conv, ssm]` checkpoint 才能精确提交，否则回退到「快照 + replay accepted prefix」。 +""" +import mlx.core as mx +import mlx.nn as nn + +_QWEN3NEXT_CHECKPOINTS_PATCHED = False +_EMPTY_CACHE = object() + + +def enable_qwen3next_speculative_checkpoints(): + """给 Qwen3-Next 线性注意力层加 verify-time per-token cache checkpoint。 + + 普通前向仍走 mlx-lm 原实现;只有 `begin_speculative_checkpoints()` 标记过的 + ArraysCache 会走逐 token gated-delta ops,并把每个 prefix 后的 `[conv, ssm]` + 状态写入 `cache._spec_checkpoints`。 + """ + global _QWEN3NEXT_CHECKPOINTS_PATCHED + if _QWEN3NEXT_CHECKPOINTS_PATCHED: + return + + from mlx_lm.models.qwen3_next import Qwen3NextGatedDeltaNet + from mlx_streaming.core.linear_attn.gated_delta_multistate import ( + gated_delta_update_multistate, + ) + + orig_call = Qwen3NextGatedDeltaNet.__call__ + + def patched_call(self, inputs, mask=None, cache=None): + capture = bool(getattr(cache, "_capture_spec_checkpoints", False)) + if not capture: + return orig_call(self, inputs, mask=mask, cache=cache) + + B, S, _ = inputs.shape + q, k, v, z, b, a = self.fix_query_key_value_ordering( + self.in_proj_qkvz(inputs), self.in_proj_ba(inputs) + ) + + if cache is not None and cache[0] is not None: + conv_state = cache[0] + else: + conv_state = mx.zeros( + (B, self.conv_kernel_size - 1, self.conv_dim), + dtype=inputs.dtype, + ) + + mixed_qkv = mx.concatenate( + [q.reshape(B, S, -1), k.reshape(B, S, -1), v.reshape(B, S, -1)], axis=-1 + ) + if mask is not None: + mixed_qkv = mx.where(mask[..., None], mixed_qkv, 0) + conv_input = mx.concatenate([conv_state, mixed_qkv], axis=1) + + n_keep = self.conv_kernel_size - 1 + conv_checkpoints = [ + mx.contiguous(conv_input[:, i:i + n_keep, :]) + for i in range(1, S + 1) + ] + if cache is not None: + cache[0] = conv_checkpoints[-1] + + conv_out = nn.silu(self.conv1d(conv_input)) + q, k, v = [ + t.reshape(B, S, h, d) + for t, h, d in zip( + mx.split(conv_out, [self.key_dim, 2 * self.key_dim], -1), + [self.num_k_heads, self.num_k_heads, self.num_v_heads], + [self.head_k_dim, self.head_k_dim, self.head_v_dim], + ) + ] + + state = cache[1] if cache else None + inv_scale = k.shape[-1] ** -0.5 + q = (inv_scale**2) * mx.fast.rms_norm(q, None, 1e-6) + k = inv_scale * mx.fast.rms_norm(k, None, 1e-6) + + # 关键：用 multistate kernel 一次前向算出每个 token 处理后的 ssm 递归态。 + # 它与 baseline 解码走的mlx gated_delta_kernel 是同一份 kernel、同序、fp32， + # 因此 states_out[:, i] 与「逐 token 单步解码」逐 bit 等价（见 + # tests/test_gated_delta_multistate.py）——这是验证后能直接提交、零 replay 的根基。 + # 注意：kernel 路径内部处理 GQA（hk_idx 映射），不在 Python 侧 repeat q/k， + # 与 baseline kernel 路径对齐（旧 ops 路径的 repeat 会引入数值差异）。 + out, state, states_out = gated_delta_update_multistate( + q, k, v, a, b, self.A_log, self.dt_bias, state, mask + ) + + if cache is not None: + cache[1] = state + cache.advance(S) + cache._spec_checkpoints = [ + [conv_checkpoints[i], states_out[:, i]] + for i in range(S) + ] + cache._capture_spec_checkpoints = False + + out = self.norm(out, z) + return self.out_proj(out.reshape(B, S, -1)) + + Qwen3NextGatedDeltaNet.__call__ = patched_call + _QWEN3NEXT_CHECKPOINTS_PATCHED = True + + +def begin_speculative_checkpoints(caches): + """标记 ArraysCache 在下一次 verify forward 中记录 per-token checkpoint。""" + for c in caches: + if not c.is_trimmable(): + c._capture_spec_checkpoints = True + c._spec_checkpoints = None + + +# ----------------------------------------------------------------- cache 快照 +def _copy_state(st): + """深拷贝 cache.state(支持 None / array / list / tuple 嵌套)。""" + if st is None: + return None + if isinstance(st, (list, tuple)): + return type(st)(_copy_state(s) for s in st) + return mx.array(st) + + +def _iter_arrays(st): + if st is None: + return + if isinstance(st, (list, tuple)): + for s in st: + yield from _iter_arrays(s) + else: + yield st + + +def _snapshot(caches): + """对每个 cache 深拷贝 state + meta_state(强制 eval,防 update_and_fetch 原地改写)。""" + snaps = [] + arrays = [] + for c in caches: + if hasattr(c, "keys") and hasattr(c, "empty") and c.empty(): + snaps.append((_EMPTY_CACHE, c.meta_state)) + continue + st_copy = _copy_state(c.state) + arrays.extend(_iter_arrays(st_copy)) + snaps.append((st_copy, c.meta_state)) + if arrays: + mx.eval(arrays) + return snaps + + +def _restore(caches, snaps): + for c, (st, meta) in zip(caches, snaps): + if st is _EMPTY_CACHE: + if hasattr(c, "keys") and hasattr(c, "values"): + c.keys = None + c.values = None + c.offset = 0 + elif hasattr(c, "cache"): + c.cache = [None] * len(c.cache) + c.meta_state = meta + continue + # 必须装入副本:ArraysCache.state setter 是别名赋值(self.cache=v),后续前向 + # 的 cache[idx]=new 会原地改写这个 list 的元素,从而污染快照本体。救回路径会对 + # 同一 snap_m 恢复两次(救回时一次 + fallback 一次),若不复制,第二次恢复到的将是 + # 被中间前向污染的状态 → 递归 cache 损坏、输出发散。 + c.state = _copy_state(st) + c.meta_state = meta + + +def commit_verified_prefix(caches, verified_len: int, accepted_len: int) -> bool: + """把验证前向产生的 cache 直接提交到 accepted prefix。 + + vLLM 语义下,验证 K 个 token 后只保留 accepted_len 个 token 对 cache 的贡献。 + 对 KVCache 这类可裁剪 cache,直接 trim 掉 rejected 后缀即可;对 ArraysCache 这类 + 递归状态 cache,必须有 per-token checkpoint 才能精确提交,否则返回 False 让调用方 + 走 fallback replay。 + """ + rejected = verified_len - accepted_len + if rejected < 0: + raise ValueError("accepted_len cannot exceed verified_len") + def can_commit(c): + if c.is_trimmable(): + return True + checkpoints = getattr(c, "_spec_checkpoints", None) + return checkpoints is not None and len(checkpoints) >= accepted_len + + if not all(can_commit(c) for c in caches): + return False + for c in caches: + if c.is_trimmable(): + if rejected: + c.trim(rejected) + else: + c.state = c._spec_checkpoints[accepted_len - 1] + if hasattr(c, "_spec_checkpoints"): + c._spec_checkpoints = None + if hasattr(c, "_capture_spec_checkpoints"): + c._capture_spec_checkpoints = False + return True + + +# ----------------------------------------------------------- 树形验证 batch 工具 +def _tile_state(st, P: int): + """把 cache.state 沿 batch 轴(axis0)复制 P 份((1,...) → (P,...))。""" + if st is None: + return None + if isinstance(st, (list, tuple)): + return type(st)(_tile_state(s, P) for s in st) + return mx.contiguous(mx.repeat(st, P, axis=0)) + + +def _row_state(st, w: int): + """取 cache.state 的第 w 行((P,...) → (1,...)),保持 batch 维。""" + if st is None: + return None + if isinstance(st, (list, tuple)): + return type(st)(_row_state(s, w) for s in st) + return mx.contiguous(st[w:w + 1]) + + +def tile_caches(caches, P: int): + """把 batch=1 的 cache 全部平铺到 batch=P,供 batch-of-paths 树验证并行前向。 + + KVCache 的 state setter 会按数组 shape 复位 offset;ArraysCache 无 offset。两者 meta + (lengths/left_padding) 在贪婪解码里恒为 None,tile 后语义不变。前提:所有 cache 当前 batch=1。 + """ + for c in caches: + c.state = _tile_state(c.state, P) + + +def commit_tree_row(caches, verified_len: int, accepted_len: int, row: int) -> bool: + """把 batched(batch=P)验证前向的第 `row` 条路径,按 accepted_len 提交回 batch=1 主 cache。 + + 等价于「先按接受长度裁剪 rejected 后缀,再抽出赢家路径那一行」: + - 可裁剪 cache(KVCache):trim(rejected) 后取第 row 行 KV。 + - 递归状态 cache(ArraysCache):取 verify 前向捕获的 per-token checkpoint[accepted_len-1] + 的第 row 行 [conv, ssm]。要求 batched 前向前调过 begin_speculative_checkpoints。 + 提交后主 cache 变回 batch=1,后续解码逐 token 从赢家路径续。 + """ + rejected = verified_len - accepted_len + if rejected < 0: + raise ValueError("accepted_len cannot exceed verified_len") + + def can_commit(c): + if c.is_trimmable(): + return True + cks = getattr(c, "_spec_checkpoints", None) + return cks is not None and len(cks) >= accepted_len + + if not all(can_commit(c) for c in caches): + return False + for c in caches: + if c.is_trimmable(): + if rejected: + c.trim(rejected) + c.state = _row_state(c.state, row) + else: + ck = c._spec_checkpoints[accepted_len - 1] # [conv(P,...), ssm(P,...)] + c.state = [_row_state(ck[0], row), _row_state(ck[1], row)] + if hasattr(c, "_spec_checkpoints"): + c._spec_checkpoints = None + if hasattr(c, "_capture_spec_checkpoints"): + c._capture_spec_checkpoints = False + return True + + +def commit_verified_snapshot(caches, snapshots, accepted_len: int, + verified_len: int | None = None) -> bool: + """把 cache 恢复到 stepwise verify 的 accepted_len 后快照。""" + if verified_len is not None and accepted_len == verified_len: + return True + if accepted_len <= 0 or accepted_len > len(snapshots): + return False + _restore(caches, snapshots[accepted_len - 1]) + return True diff --git a/mlx_streaming/mtp/qwen3_next_mtp.py b/mlx_streaming/mtp/qwen3_next_mtp.py new file mode 100644 index 0000000..c47bd9a --- /dev/null +++ b/mlx_streaming/mtp/qwen3_next_mtp.py @@ -0,0 +1,95 @@ +"""Qwen3-Next MTP(多 token 预测)模块的 MLX 实现,复用 mlx-lm 现成子模块类。 + +前向(与 vLLM/sglang/trtllm 一致): + emb = pre_fc_norm_embedding(embed(next_id)) + hid = pre_fc_norm_hidden(主模型 last-layer hidden, norm 之前) + x = fc(concat([emb, hid], axis=-1)) # emb 在前 + x = layer(x) # 全注意力 + MoE 解码层(内部含残差) + logits = lm_head(norm(x)) +""" +from typing import Any, Callable, Optional + +import mlx.core as mx +import mlx.nn as nn +from mlx.utils import tree_unflatten + +from mlx_lm.models.base import create_attention_mask +from mlx_lm.models.qwen3_next import ModelArgs, Qwen3NextDecoderLayer + + +class Qwen3NextMTP(nn.Module): + def __init__(self, args: ModelArgs): + super().__init__() + h = args.hidden_size + eps = args.rms_norm_eps + self.embed_tokens = nn.Embedding(args.vocab_size, h) + self.pre_fc_norm_embedding = nn.RMSNorm(h, eps=eps) + self.pre_fc_norm_hidden = nn.RMSNorm(h, eps=eps) + self.fc = nn.Linear(2 * h, h, bias=False) + # layer_idx=3 配合 full_attention_interval=4 -> 全注意力 + MoE + self.layer = Qwen3NextDecoderLayer(args, layer_idx=3) + self.norm = nn.RMSNorm(h, eps=eps) + + def __call__( + self, + hidden: mx.array, # (B, L, H) 主模型 last-layer hidden(norm 前) + next_ids: mx.array, # (B, L) 每位置的"下一个" token id + lm_head: Callable[[mx.array], mx.array], + cache: Optional[Any] = None, + return_hidden: bool = False, + ): + emb = self.pre_fc_norm_embedding(self.embed_tokens(next_ids)) + hid = self.pre_fc_norm_hidden(hidden) + x = self.fc(mx.concatenate([emb, hid], axis=-1)) + mask = create_attention_mask(x, cache) if cache is not None else "causal" + x = self.layer(x, mask=mask, cache=cache) + H = self.norm(x) + logits = lm_head(H) + if return_hidden: + return logits, H + return logits + + +def mtp_step(mtp, hidden, token, lm_head, cache): + """单步:hidden(1,1,H) + token(1,1) -> (logits(1,V), mtp_hidden(1,1,H))。""" + logits, H = mtp(hidden, token, lm_head, cache=cache, return_hidden=True) + return logits[:, -1, :], H[:, -1:, :] + + +def mtp_advance(mtp, hidden, token, cache): + """只推进 MTP cache 并返回 hidden,不计算 lm_head logits。""" + emb = mtp.pre_fc_norm_embedding(mtp.embed_tokens(token)) + hid = mtp.pre_fc_norm_hidden(hidden) + x = mtp.fc(mx.concatenate([emb, hid], axis=-1)) + mask = create_attention_mask(x, cache) if cache is not None else "causal" + x = mtp.layer(x, mask=mask, cache=cache) + return mtp.norm(x) + + +def load_mtp(args: ModelArgs, weights_path: str, quantize: bool = True, + bits: int = 4) -> Qwen3NextMTP: + """加载抽取好的 MTP 权重(已 stack 专家、已 norm +1.0)到模块。 + + quantize=True 时按主模型约定量化:线性层用 `bits`-bit(默认 4),gate/shared_expert_gate + 恒用 8-bit。quantize=False 保持 bf16 全精度——草稿质量最高、接受率最高,但显存更大。 + 提高 bits(4→8)或 quantize=False 是直接抬升 MTP 草稿接受率的杠杆。 + """ + model = Qwen3NextMTP(args) + raw = mx.load(weights_path) + # 去掉 'mtp.' 前缀,映射到模块属性路径;mtp.layers.0.* -> layer.* + renamed = {} + for k, v in raw.items(): + nk = k[len("mtp."):] if k.startswith("mtp.") else k + nk = nk.replace("layers.0.", "layer.", 1) + renamed[nk] = v + model.update(tree_unflatten(list(renamed.items()))) + if quantize: + def pred(path, m): + if not hasattr(m, "to_quantized"): + return False + if path.endswith("mlp.gate") or path.endswith("shared_expert_gate"): + return {"group_size": 64, "bits": 8} + return True + nn.quantize(model, group_size=64, bits=bits, class_predicate=pred) + mx.eval(model.parameters()) + return model diff --git a/mlx_streaming/native_moe_ext.cpython-314-darwin.so b/mlx_streaming/native_moe_ext.cpython-314-darwin.so new file mode 100755 index 0000000000000000000000000000000000000000..e9064e957f3a552f3581552bdff9bb05ecbe1836 GIT binary patch literal 1805288 zcmeFa51gD;b^rg&OqR@Uh!98&5Xc|4m|Y@78Zkt~%x1sx zz^I^6VT3jo+Oo-N*$r5*)Us1c0oQ)Tf5?Ir6)UsZgv=&dL~Plo829)7+~?kz`^?T} zHx^p`HF>=@^ZdE@+;h)4_ndRjJ@-D}|F2*C_P`)W1pG_$H&K;n~-~yYlKQ*S4;H-}}~Fz80|6@P2xB z1n;m2T%da&-s|mi_tVIhWBd=@2u0J8oKx4IXLdOwY6={ z+H0=7dZm3(4KMTO5q?K5iwd}VSHBuI@K#^FroxbFc$W-B@ZNG%1VQ&H2du_#YwL%v zUU%iymqEt&t>%3-ye}LO$;r%YR9W}cIQd{DcO?Vo)RuEwTC+>vwJef8CvO>By6Rpy z>QAr_<_|usuJls(wc%}ux*G(*^ZR)KFdEG0;DUd{=@y5#;nPt+Klr++yzYH?(bLNR z>gSJuAK_Jb9z1HtRmMK6x_AB57}LMj*2`9`T@hVJ_rj}k@8CV@1|;zIh8N@Yifb-k zaarp}R$QY|R*m12-;D5klWjuxKDlf&MfatR(Y^5UaeFfoDLGcT^N$OHdG`MN;|>hI00QcsbW-}A2L_*< z9t2Om{dK{kQziz&_H6$2?^d$e5tTomUspbvxrU|^!Q5rntyz2dRddg}^3rQoTyx#rcU-gL zs>|1{zGl^$x$nGu?TQZ^|33F%?s=DAv*t>a-Q1SwRT=Ij9^4?f5;WhK*bCfutp4!T z3^G8~0-l{)c@JFm(}$lrp=(@_+$-LH^5GTNto_sM>T52q1;1r(<$VqKK`@S|dxhW9 z)^_;^F282Q+7aMa-dp%FK4qmjfdAn&*N98{YnwCY_!BIz)Zdf%b9@SdvrtX(@)669 zU*xgZ&?#PTd(;oFoz<7VzwP)Izg7?&$vC*e;yL%-Yhdx*Yd?JX`#*g3wO6jVdga_x zFJH52?dlKCeZF(^j=$bG_dGhi=D4d?TzU1}<3Dow)gLiE_~3PGuULKcyyH*!;C-n{gZJhp4YubF z9^9NdcyQY{gRs9R2)EymDC}6DC_KL|QP{aAQP{OQQ5aa2D3tE5TTr_Dv<2x@FrUPk z$w6|%Q}aHRyrZ}@2xrv=znm2up8wlhl6UA`acL&pOq;V#Zwa^XyXd}5_zueI-OOO- z4@#+EtICw_55iI^v9+{3QJBm7=Xt+5b@pH}RhKU=3BrLOxTClv6V7ySd44t5#l=A= zSbd8#VX-Q&{k=6rjQPTf+?pmbwR#F8>OM4sYS0(_XEej z@8rHcclKabZO^lPn3oCLbBW%UZ68YOTMDJ+K|#-k)c?5_$K{#Az}!sOl;B-QvUfA@ zciB2u)CIStZ#*WShQFnaLD-(^eOqzhk^lb@}a2 z*X4Kg)a9Rlur9x|qb|SO;!{kG%lA?4;6!l8X8POSmg+6e2*MBZzBD5f1}#Cp@0*$M zL-wBjhNlM`o+@1cj>|LQwhN$je=c2A z;5*Isahhr8#p2W;yqB^i#z7Zl>fx~|Gc;uQEL_!25qw|IuV_0%zqv&3QF>0l-=JS) z<}ZNLmAiPb6PZpkoqX??;x*#D&zIpkN54l*1_U0cv*V4uNsN5*9PdTs#?5a8rm&3sIJkd$I zhxSU~c{**Hd+p=XfZ7{fCg^qPL8kbCNg-vcSi)z@zv%8uNZKL+l;^y3@#HTqG5 zMn&*Aq#?K?lL)>l9?V4V1`iyOH~RM*N92>oY&fTpXKlf>{I@LcO5mHW2j@w_i!q#X zNqA6?{Yd6_VLNs)PP(8&CvvnEKDMW33~t6&Y)MTY?8qH8_{G#wgI~%WIrulJBL{b) zM+O?931hnWT#zp!Z+B0m56Gm(+2&O1;19{8H+o!N18}ICxL$!oi=< z&V;)gp(A==S1QP#g>G3=7ldcj2jMBe*t7l&_@5jT|7ZSj4Z8JI^JjNc>B{2NJE)|1T<%1#hY2JYx(zj(sf%l=~g*voz-J-qC%mwNs}PebpE z@chL&LHM`O2zTa!@ZWC;!e6Wp!k@1T!reQA!mj?H zFz{SZC_Nh#ig#zi4WCTj(Z`sS9(v2r&?aM3cV-Hu#gPqsJI{*9n|Zz%8;Jb)Hn7>& zIUTt#+rT#PTK4EE#YSvkHWSKExQKR&*oHo2*1)1~1K5a|4a_F|HLE?_ZP3Z*NQGH#uF|!#m#J%Y7gHD7N9VFoyo=w?msW28@lnD-(9Rdu&tRrcC(v z?*1|EZ_k7excfvRFCXJRci+POM#iJNzmfZ!GvOEA{bStc@EzQJ0(@@Bgn#MoTex4J z3BTa(Z{&VmCcMMmm+&jDNMLj4L6f~Isz2n}M|QO%**np~RlgeSwR|$^A!AGVRlCv`rvQJ5a(mX-p_3DN*{9Ab zJ=~9cZHJD<8_>Dnv`G2`oJ!#7>)h?g-wt^3%A2H5PeY$3#-vY!lOuRz(TUsMI$X#6 zX7$5ezg~)6xBkFTc0%t<=$V)11mXSQ__xTzm%-~k@VgfsvlSiFbuzkUVW#kvM#jQy z#s|9Q4s^{O=$gOB?!5G15E^}RKmEQ3S?xgApmX}sIor`WJ8nnUpmTO^L?7LZzR96) z&^bRx=j?s~eS2G4qC8pI;i2)vFo6pXNT*c(dHv;CK)~->8bdPwdOFR)zO@&Z#?{mSIj4v zBc!)>p}Pm5OLHO<)+czT>pHIM0^$Xk@bMt{d&A!p{NjoaT!f7tPGK2f%v(oYRTsLFTa=OY}j!EYH8P(fr52qaWotPZhm3Gv8DH ziM!<4{%HE4YxeL&*TD4q#FoZU=Z)<58pfwR>mTXblM3>8A*VMhC@Hf}o6ZbIXHIDk=&o(<}Y~)@%^ZNo#%5y|B z4k(`vh&j=JGuN%c(alF^t9@@Sng}AC1VeL_CA_cI&PZ_0T!%ggzM10y=RG69kSsRU zN3yB-MLn^+VB)v#EW@^aGUe*_QZ6NbCDpa3Cz|_jX3kjV0rg7An&{QZd~g7nQ(KQF zY)khkc*EA zU>eyLKkE%$y!n~!uaSMnw_5e7dsr?RAA3E+b;maPR0Q8X`jEuNMdQ$mf7GCx=Besm z`WGL4S~6UOChC8X2;O4!1oxtq;jg6)aET|j4zkzY%4LNvjwTwr(D}vUrn`?A3vK?c zx_?!45RLw(x=dDbFpjaA$ag|Z;VD_zBL9{7VkfZx*=XWw;fy-|>KR|~elmCq-kJP5 zo=c9;^6#O`*}8|Gnz#2$Hn8*Tl{4{%J?l^H8p>M#wg#@UbMH`Dco&O*U=R1KKY+MZ z!o;m2e2H}s?=bpIJm?7WKa`(-SV-Uv|&vGwPu~js24x8=FYqG!`y0{o4CD zF!~W#8Vi!6_ff~la3a`^uXe6ze4U=rj$}yloEqbz-J4*mR!7*n6U72^RE7B5k6VPiowB*S#VHU#hWkUd7P)< zra7x*fimyry^$Yy{f{?Rcx`wZY&fTBR9^7$6emQk~doTQbo&HG+Q*xm;jQ)k+wQQ*Bm1*hG$M^lDZ#XL;Zi8#k zU4DwMR~+8W=J{i0ZVLPl z%UO>>{v@X{I*kCs&=z`Z#DA#PFX(FIGZQAT-RgS`Uam6s0^a}1;1Gl#v$z=goMqNM zZ(H8EO2KN*>yS!L^u4AZ{}k`Kl%G&1%@a;baKSNhT)JMbI3R#{u| z$H-&g=*wz#S<&*V)n#hXPyUV5)6Um?lV~wK$Hcar z@2Gjau_v;-Y73ovJ>`rq4vxkLfIreL8UuRY$5>aHSiGx>7V4j5ngb(-`va8q$CLao zGp3vkse_l;5%Y{|jWcIUDr2nT2g&d0qhIaxaV>b2r2FWL+K@e8t!LKH?aT5Gy>KDV zqIDOozlc8Kt?%Dd)@(|t6ftv;rr_U+?;`(XO9$7)&Q#y{7PR>nz&3qqOyuQfNOmNH zinU5t1a{rq@jx_vEAay3mqF_-lobqDPh+h!g}+a^=z99#&$a#yAB|m`ja{QH^x7Sq za4=cdqWzx<59CDkM2|1%9$NbAKJvH18yGh=__x;B(c1P+^pHN3Y)URU;RD!Kf0{Nb zzQ9;%)jpZ*HKOX*^Qs%wygH?D^O*f}&R4=BJZNJLC(hAQ{sX-UX zA!mgQ*8x58wv8NnxYx8%wrx`uxT5XM;89TAa7t4e9^q@wxFn8`&s%hvnd_Bxjd&|P zgAA%}nZH%IIlJ#{#PmV=Or8vR<>hNRd*@yfE?Fe5dPlpnne;(6qn&s1y^P+1hQ9@OS4T0Q7tv8Md+E{1$(yll zv?Y0am+%JmNVN6q9BImeC-n~J+Kr`jkltM!fAW!pB9DhY~I~RLV@XI(@+?@Ym zJE9jP@2c zBlpm?p1EODorhCTMGw>*JF_wiI=!*;v;VR(1`e@yYV=Jsl7A_gjQphvzaIxqH4miI z(8b|Q_}j!7)7)P6`cuqb6U6M~p9+rRiI>>1p!ojwQ;|!9Gws%jyDP3{#@ZYO0Ko7Y2xyHI%x7ciBkE*M@419^nnok|?;iEa6 zeB#KLOYgqP#fYi%X~mKG)fhWbWw;LN@Sot{E}k{lH*Bn*7_}!|uXbl4J4vqP??TJT zs-x%C?@!~s(bv3>!K>GtdW1NErF&UMX~)@#nC$uGCdTbgtlS7ZbAG3MFX5l|;<#T% zx5`(#5qalG%D5h{A@kDD(zR|5aV~Wty%qUfF~4pM_03##yyT@u{oHW>Y=*(nvrm2- zp4_B8`|;O|o0(Pi!};v88Pek}uA_Av-HVs%XRJS#4?!?kj^EX?+eXH)FYma1bo=tw z>N44ik8ff!(0d!SaXb`W1NiK_kij%D>E=YkZpKIf`*^Owc`<%@W0030jBPwe@2p%F z9=9}*yeYne-6%AX%WmH5UcQ9ZMzAJ4y5u#ThkG*N&-0`!-27F3!|*td=Qm^(=Sj1kmSk;;^(f*xTZrcZ<;pU&Ew&DHf}$B%vek=kgB{6x`1u`i7yr^7Xm6W&@A7*3Pg z^Fp7l^sl_ux(&HsP!oCj@0;aaXtAGY5acCSTd;jx7Hm@Yzums z94YS8g4>!;zHnY+T5SRZrZ zeUu{_@klY>Xsy)6cBvb_o$+ojm>gRf3#AxQ}Wbr_AJ>jMH9PQMHYX4-Dp3x7t∈)KgM-t_~zD`?=<*$ zI;clHjOQLGRxcf(*jRf<(EEAnn7j$rj!mo_{cLO{v2o?C`!RdtZxWB!*j4QO4K8jE zEp7#l>bbhIt&)=gt+!xTOw3+(*RDmY?axw&F*aG(pEkJ1a}f>^|FY3s!1#}2T%X>tC+=FdWo__)v9XG0dt2Xr*MCY! zM7eCt0e-3V?YpVt&$FH0bN*bD#;xSs=xl5As`a0xPi%hHGdBNcTAALS9XwBg1=JYqR4CpEnl!6i%WEI{H57f}BK>c5xz@1;Kbv*zLB&xI#*&^4(bKPM4{$JPbJ zAA>OK!O6w~Vl7GK((Rv;LjzQAioS78}J+ zY|BFJ{|S0OMSbC2Y($maNKlTr0eh zr<$)?9-!-MDiJL6lJxPr}71Xiq0Rlj__I)%1kecKCsVtF2-7!1uY3B^OhxpTm6Q`ue-veB?az z<;)S{BZtziXi7g#-pdj8zM_jgpUU<-`1L2A3a;P$t4!;Yzj}0P@mFOUG*pj<=~6J(5k3O(7J zI5w}G^m@imfV@jCGxXi)Jj$HOwfwx3n1|Ajtt0KVy${{K8y&uDHoBuR(j9LE=A1h8 zIl8=QB6|D#LEeYGXZ@Mvoknwd?b$cFcs^qv9!R(A*#P=^fM>$(Jsw`~heifRbk?WV zp_h9qy0+HX_>B4iZEulnSW(H}GQPk*jPH6T1=biB2=QZ?sHBZ_Mex5$BA&;_G zUz=4!zr6}&Ur>A_#eRfpnXdu+A#jM%vWn+!p7mE&R@-Ms`swe8hs$r13_Wh!`!TR2 z&ytCw8555i*q%R{jq19wYvkr4Pi}xGon0RDRk*kqi+t-b>&{eN?{8Kg!k)%M+2=5h z{Psj|iSc_Ma&j+n!g(tp=dFaFMTWnOpZ4VhdmNFKU=rh+@s(tpr5JDH5<%#Ew0m-g z58jtL9AECR!L6yo$ibRASU-{RiEfa+J{B0emIkZ|u-}<|fr&c#C&~Oe#+1faX*T_& z41WdAGbwc9yP0czrcQh~bDh=m_e9r=_1s>Qm(`g&KYBh#&&~A$_Tv>c=a!S(Or940 ziQ;I*Ip9QF?Ab4%M+*3ag%ZBVcJ^%UV9)0B*-T+4dnA9cG!q^-g?*PDnZlP-)4OL) zfq#dW^-TeNQ#cd9;aGB*zOH03u2U*Gc=vMF&Q{La z>Ef&%>#IND`|8p82{Q)oqfhstFYZgVcHfses=FOq@N@F_evTY%hc`Rm)${OfC%oK^ zUd4Ye3_KDni2TC~jeocuI`4qq&qMc}(EsP`oyAWt?0O`#pbKBI3%%5px~RJg`ftXE zRGzwgMRdzB`X58TtWRE(&KEc-24igodRpVy$fL{eKwdQt&2!{g zK9jDs@5B5u9?UOe!j09~c}6|sd;8I=)%2CkWIp84SbH^&qOa4T??upe8h;me-)x6>th;N?VJCGm?%EkUMfgxc?{vbK zPUb?Lx!3b|L3d|rYIkSuJg)V8Irm3FTg{s|&jK1MhOV)Q&3>S{Wlqdye_ihzqxW4s z@9C~TAt-d^_&t%|EWZo*P4RmQzdh%7*SF|Bzo+rL=KPb6jn-Eh>SrYu1nkXyH#$;n z1)CH3ZeT~($^2K#*Qxy9m9NL;|IuEjXSFPNd_t7xr8@QKTg^SvCoqrK=0pG-t!vJcskf1xvHw(wqXo!(LVn|XfI_zh3J!WdHhbNMa9W~>?5=o#%rHL%a0 zmS62)&jvPnll)coIVbaX#b77%eLNTJ?Z8z1Q+(LtE$pel&L$$**G|iy?_e_)HC{^{ ziTpO+E2dY$JH`I#gn|7%!RGxN_+`zFdHunI(jlGMgS(Hex>x^>q#SEnv2w!g^*rZ$ zo$=?uJ(%aL?U7&0+Ds}4_omZwl$EU=>KUJx9d+jhMf@F?Kc2psYh+;D3swf;;maSU z9{fDP_Fd;ny^&}B+JLV|wGQm`k%=q9C+2S5jH!a=!P;Lp0DVoJ>lw2xL3HjDylSco zI9q*z{;@e+&)$8&e;HRx$Hski!IZyqT;16Hoj2Din>qkn{(5Zr zRBU-&wB9IxL%tq1dvkVrx7OK;jJ2)Vi@WRPi$69kpO$}Gn3g}YIrvcGE1$Ykd#SZv zrZLuM#z~?#piIy{Eq^xcO6J`BJ-u0?Qq*iY*4dGnF^H~Mfk1E&u-l(|Z9kl8XE zv7k=rxDmJ?obKlgwjnz8yK87wbl3QaQ@I{(9+%L0LuTB2zIv|q7(S-P*+lpNCXZDx zoIeoTGb7!Vit_WcH)#rVl0GouUb?)Fz{Oa7nCd$gT>z!FAQf!g^ zNryUGBYQtvle{D5f8?h18a-&wc$01@ZcK)KHz&hwxg=-oEecCtL(XnqWajRD4-pem zzivpHXNtEH1iPc}p-fm@&$Ha3@VDHTzQ+B!B8iQp{b}LA!gP9lzrQ|K6w{#d|a1 zx74o=+piAlwO;dH^DLdo z43;i{_7^M)zt6qSsWNqWU%H#W)}(o^SY>Hx5+60g`76+F=^|{ww82uxsmzUz#E-lH z*}HJ??*!v6i`!k)Sx(B}bi(myHEzn__I&P6i#bTEz_zMH_8GFy1oB{;W7aN4=w zCY*Vff))d}0~h=TmVl4^w%6M|fokVg+R55>el$Hl4I8C)N=taRQ8*kN;qd6R{AByC zm^(PnSwZ0s)J7xMxgh+hu5V@U;YQ|hH*ii+4%~p-PyLSyx0@}UC~wLFV~4@<;QUYY zp0YpUw{$bO&WFCs7lnQE7lkW%X6UPXlKzpwCn3co|CLb@eWD5c=rbI9~~#uDvc3asbA+0cjk(z;0A zT7`GpLHW{m!26p?a+x{*swWwi9!Z9~zq2UZ_02`$Ko8{~SrqPmHW}`ECK(Pq&HL~3 z`z+6&S!C>8>1lrJ0-p0LdqclY7i{U7{@3M8&n3dXp4a8mGlBnXB3$_*^V{{7uIr%N zdd7JI9;E17D!FywYqar@#pgk;?+2f+fzLzW^C0cs4?erWXD4{{^Zq%0ck_Iw#i!rm z^IU{aAAS6={5s@$V*YVxP+aBYd1Bs_1;z;mw~6^TfRAwa zBYsP(;L9_KXiT@#e#e1P`)AL{XMEf)5Z?A3xGiwF^-stbmj)rp{q&#fWwfhyTEXoC za9av)%fJn|{nXzq+?HG1C~wLFqd|3eKcx4R-NWy|a&VJg=tzVC@}FQVB$5A=WWRAj zehGE-KHzzR_vLq-lXj|oH?Z^Lg7pVAWWUC8rgE%@yX`K;b8w(l16?&Qk5x9MHh zzRU7%egseTPSm@F_T56>&8yTqUhht}?@s33+{(MzdN<#`o6oyByjvRKVq}Myk&(4| zyqnFtCDFUXRd23+H87`bB3pTuYBr+?e&-wyiM zhupnr@r}yhzp4y<-A-S3&{vfy#mc0jGW2^p{oX;p`;bwmtA+mnwB3c?@9a>LrVm2?}L-o&QOdSLb4{ZtM$1de8RYM2hprfBGp9wiW z5kCDP##Cd(!@kB0I%a(ST*~*2r~Z&~C0mFcq|MypUnna$f;nwB`kUuJQvDfG{o;&F z_*aYv#Y$C9anaH%tm(9+EM|1@Wk2AHGX;cBwHvue3kzk9^GW0uk@e8w=MSh3jaC0{JMR9f&Uyn zZnMwd?LUXNoO@&1UFtuF-$&c$XZX*tpNp!>8+|AH-!?u!@N6Qo|IhJ@ejDiLSN4Br zBC`LxIiFni{{?(n=y@ryp0l(!^ufQl$-+Ouwp*f{;;ZsiijCvV*`41TfX{~_Pg67D zA36MZH;rpkegfrzRqQ_?uX*reDnCK;$GJeRJns&2<;Mru=#LlDKVJ9rK<KEkr8GCKxTDR3YC+~aEU??{&7SoFPA&jfjb`5RP zyw-oNd98o1@wA6|zUH<5bMfq};8E?jh&T6pxZdZkSHZW`@N+j?dGF?*#BhEY48EbV zM=V>3d8d){z|Sks)t=wJ{|Lr(N-;1m=U{}gZk_EMXcaF%a*k&d$Gdp2ePz(wNc^cB zM}b!*WHDf$_RK`IM_YUxLQaQ}^P#Cx9I}{u>}`6lwOqYdtV!$7TE7>+1(SZxcl>Ws zj7xqk@3cQb&!ar3N>1mUKPPvss*h%k4mv2$xZUM>aNlfU45N`p>v9ahJ}c37`l8H& z^!4MHX&zf-FKxL@PdvV8)*Bh8?DO69s6A6fW7On3Gag+of!R07z3XRejmNfIZo|VP zsPkeJ-gDtse|T_6DOJBUI4V=wqhZU4e#p@0m3zrvp3wZVMxwQm8SGh_aWH?AxeqoR zl73TANWYoClf0a^TwLBn6XqMrdyAc^W!5#vD1ZO$vR>cHwCbtwL-=6>C9fwiZXJx3`vQad!eN_}Yj9ZpJJg@xwZC{+{YrkXVosug z%L!A?$1-HIoi!_sy$R5wotU%qi(>K3{E{D{c#X}qI_f0l|LROq;xp!qM!{%My$6F{ z)|I2&Zu+(5{-Bq7CO7^Fekbdiaj&ti952;-$n_hWr8UB1D5LjXT$>y;+6ia}xw>Op zW2T^YTg&Y?Y}7l&w`TKxV0NZ}?XWpr;JN0aN=_H}Jh3{m8UDFf>=z+##_es1;(E%h z(763)&SinO=d6RaTBAS@6--`P0zB#l%{h2Yyf4#c-jOFMPBn@0k}>5_cOpLlbIRZg zN9K3h_BH-D^ZZlZz3A=3xl!jDMRCL16gNBqnCBZ^cchIGAK4wnP%HQC!2B}{vvb~s z>bt>F?Y&~3p9YMmFPXxv^iTP6^h;|mgNZ~2*|>Oc9R1_Gh^LMQC(RG)xIRvO!q?RI zGSYRySNiIrJyw4IOE&}y)c;=EQXE|U*Ip3Q{~sNhzsL0#dGF)-8Q#6$f5EBl|4%4; z*eLxco|J#Lg;~16?1_*1PkUdp&yBvN4}N}Pd+x%)8?R2@F_Tzft=O{LA0axLeVphK zU4MePwmAdH+N0xWBc?lYaUCh0%f1Bd(T>~KY>jv`ymaSQE9b}AkyhID?TEq0vm?rl zb88T4FNTk8N4h$I*JT}91J5zxSfk!~BlNA@mc^lbpOS8}XLb@(S6-g>_?cWZ`Xygh zI`$0HFZE}5j-C2hhSyV@Zj-9$M(o+p$;bonwH`bo-Kt@)r581Jbg#TOmC^Mr?wWU-?6u(jCD+C$qMYvUWZ!}471QBWG9=mg zbHPBCv?i#1C|cK7U(Z07)u+F8r}nEn?)FeqCOs>7QM5S(9W&G#EGV^_^Jt9z5DdMS z{~(>Kd%;Q91IIoWjwNsujEWpoWYybWaa(ZNk@>~)QSDeTJh+H`@H!taJA6OE0C?{L zcGE?f1-nxXTX!9_;ha_YcIg#|W9udy+jJ3mOuS1c8X6jm9ceJn%CJ2775(Lxr)h7C z?f>KGY~iiBPMZ8$(Mx5}+a{kfJud3I`lou2IlAdy{Dm&H`(GIc)$(inar!0wxftHW z<{OqSDpT%*_e}HFM;dK3n>M|akUKRV*6SoykuZy0NX(R9KMLc7qeWTDUiToGAv(kT$)+4|-{si#F zBh7Q&ykQ&d4&zr2-vB-4a5LkKdTH?bmRmoiUOJIkptBf|(t$fmXT zw03`r>qnEpn}wJB!!x)ywpKpwVMpb^@c|plp`8`PJmmLY%CG$13n>dP%zR(*F|&R< zBPcB9xsk!e@(Ipj-?!bbL%i&Dz#D1Ys9pI;X8c$lJVD=lKklp4SD$0PQw^HPrh>zm z<101GW=Hq^%wA0JW?dq7p1t;gh^E!JzPA+jIefjL{);bf5Z={&?1_&DBd72b*)o6s z*UzuN8k)sq!tze`NcN`L(24Wwv7xcENc?hjdnxDZW^}A_Ov*f+)e9 zbvi&}yaihff90P9_$AUC`+BaI?#dU&jzpL$^vuBd8irtlO8lA~*WRS5V z{VJVg#-jDrBx@Sy%EP(>9gv2udZzDJXb+fqcA9b`qB*+uHhjkF578#c-<|Yfw46Wr z@5T=W$0Yn3rgv-bqI@3Li`SQJ<(Z7NpvqPne;&B9|G+iz$$dSK>({GBw~yqr#QY9N zck!mA_MzJVG_G~_`d-Ua=>gduH%BQDGl-1=UoL3(kHAT~RCz3FKS(V6q4*u?8h@@5 z*ENE#az{|VFe($yG=0R%n8NnE1dmra}i^JOUc>p|z+XMSh_WS!# z_TTrR>@yQ;;F}$;3xxB!_lx(?Rk{G)>s%kk`;+Oz$rDGGq$l(($S$K7r%M++nMN0w zeF?~q+vju(au6H;DaH84lVi(X5?~u&>Y%{neZQABm1|r&XhXs5C1ZYDpW<6Iic_M8 zv_HzUGlkd>IP7B1JD~U_`pN8n2A8eqBKA3;tB)$d;<*Hix%Kx z;(emMaEhIs$1AUv=+6J7~Fnvcqab12_%3_P2a2@gCY2q&Dz zKHL_{o)Sc5;930w;GIG2{514$3o-apfUWTh>|OYh1I%;l!5!U7%#E|1_*+G6SAViAo78k7uPqip1FWE%mq#M8_@=DVAt1!OFj5a-^vwj3Gaa{?-vGQr<7|a9RvbOTlRg z-{Gvn>0xjZep;_S{>tk7)j7TGeA{_5-BAAKB^B zcy!;403ElCeoW!--Gh?Dt>k}prKSPn2>P!4&kF{Zs_!b#8116%F7^m*Wp7Zd@19=L z+K%F9Ez~>R^_{U&dgo2&Tyge2*93{#)1RQ9*uMX zydb;3kuhhl>u4K3o9Fdh!$Yly8~Q+7`S3-?{gK3R#N#xvoukLw@B8pv`5;fi7uHM) zW==de;>oToaWG&jr%vl)8sqY#{pSfir(BY9_k*92Yv7qO;8LzT!>enbm+NxhuDXmB z$-=O@@V(q_jkoA7*a&4u*+CfRSDzpWUCe5Zc+ z7}?m?!8;_Y&?~k^&=dDFoiFQrz3e1cr*_;%ep(;xXwIzoOgnqu6+a+{Ac_t1OygXB zzVxuJ)wjq;^I&4<%f1KjR2Eq+3{W5WDKMT^74PxBk3#wC7|eNBNKYuP&R(cJmpsk2 z_`l!)@S$C^&L8tbYrxPvWW+MyF1+8ceP~nq5-W#>o}S;H^?ch&nFZzfT~CG1vNa~= zkVQX2Kj+(<_=kb#(Z%Dj>iBbB&2vT1_Qg?~nrE6h8#sy=>Wl7U{I1bw`RR(44D*{k zoL!7W^Fo6!bsW#9lGTV>~(e!0~g_v~Se(8G9|CP;=fH6Qw7#_8gtzz9AXO zopgEo%@JM5Su$(xDre}uF_N`e!kh7L`oVZPmV1p$<%Bc=S8}Sp#=cRKt?)NZjB#(d zE!AVmv{%jF8Z>tDAiRzdj$^1(i!bbrv^@3rvX}O1%Af78aHu8APA(kIpC#Tt4Br|u ztXP4QBi60Wye6{E$CdZxN^j}g6PhEcOhs>0){Qi-CAZ?M@!^;Q=sZ(X7WzH?zDmBB z@G>@U*4(^eLBt>S+NWAHWld~2O>5C8H(WnyoPE7q-_sc*ly&)L|FE(uj?q&&%S7}y zF|fwK&`JA?iCt`MpB>5g1>y%Z)LMtmo7CQ5ttW0n&XsGT{RQ%oSTi?!Be!9j+BN+K_fEAnoAERU_|X0blrv*U@aX$<+zU7H zOEzP>;27DioMEE9_Tpn_4|~F?_pF1#o}Np@%Nnv|`VYJ=SKkk{SW0H-|m^_;$yfF(a{rgB1TS#wQ4d^;{$TU)MppEs|L_g6Rz?s1YK zV8?8*g|9MYJbL1MaJtse0KCq({6|OcroZQ^k9)A&vWE%cnOeV(#jk3>t2O@38oj~K zqtzJeuBXkYZ(d#7HzOwruYdOWXnn4Q<5i=yQ%^hiMkW_g_>^gytIdZrJZ6lR4 zE80XdR-sKT8@UX4rYv-AxV-jyU@h6I_-~#JL%W)NMb5@D=2;JYssh)OpPu1kT;~}` zcAJ20)^fnVDG|}Xj$iisj;`7yIM(C&W?;L$9JoaQ~6uHl{_WWwf{htBP z$^BWE{SV1~HnOE7=~>C2^t@y*k|9s-YRKxVQnm)qsPW0URtAy*~+Z9 zK7oHFUru(no^fOJAio3nnu;rF9l)){_E|gJWOA@Pxs|=Q=)Quu0M^*7%!`EkZzLSw|RWfYx)PQSos>` z)vdwrk4HrD2*o!KM|Nw)BJ^Ckzd9B%Q?iac8(oPUH*-JMScJxt25l@zKR0?r5v>4fYa;FmZ;x#3Amd{aSH|hxMG8xgUr4rf?+=p?HEHhxnnM z&lx=q(Mvg_`#PXM{rM^Pny0J(vR!?GW5&bCaflx|7{npoE?$nL2Z86uA-+sK>0=j% z_<-7G47xbP_r@59Nbf@&Vwu`-w5Pv*9O6H#;t*^05{FQX)%XCw(E8gz{Z<+)a__B{7ZuVL)rXT|-o z2aSV$jXm6ojEqH3{VuVGj-UPS#2%KQKStBL3u~0AImiAVh&`NBi9Osll3m=F*u!~V z>|ysMul^itv~j*a-Z1+7TfV#>r~A>ry*BZNn^ufI7Cus4re}CusFvSZnV*f`Ul*&` zREamdDzbi;7jJmeYboophxi_4`{>y?-;PU{?k&df`&M>uxEAqG6x)>j?;FQ@$H!;n zA0sDZG~Ft{p&}c9z;kzx$Kt*0EA;sNJ7eGv1<#(iUfI`uCEs{<|Gtt0v8gx8XG0b> z&soBCEPj-&(oui0|zFfZ+%n@k*JFiQ4`#qkZAHi#|-{acW(fx!ERF|ph=e~~TCx5bkJpZqb zXZ9J4X`fhixshTs=$3RHHj;G-#e?MIy0wW%U5p)_727Aa;&+@{x_d~OkZA$F`c`3^fA3|lrld3*q=XKA9^t&`A0!wxNg*Q$ptZ~7t4CHN_WUE zmDipnL+E1FyLAY-7`Vi_RF)i6vo5c9$MH_OTr#M*n#wiRMeu|} z++J4J+7v5OUVzbK^oz4q^PEBR)KbA??O+$@)+!zbO}6U0q(MuNU(P$Vqq15rOA8;? z<}T*Bzy788q|2XES>-aBdf=P9NON@I5&s6NnKM!EHq$QgRLUmE`M-g>!XdIJe4~K( zTJz8vp^1BT1o;%W-Ay^gb^l5AdFSJ;^$*v+_KcWwb%Z;4X*a?6HT+cQL)WWDOaq_Fc5D6XR#q_?NBLd_=jYnw!P! zIb}zT`PjZI=fu%L{!n^i^c~xB31!!SAelym#H++M8#=&C$>?Nc!4mM~755_-nE zEWgCNhwX>2`K`{ZC$C0U*@ucOO5Sf&AK{1AzO=R_pUvfFnH(eVdP&~{LO*LQS7pvb zzMDD2LS;MLxf(pz`Y2~DOqTvWoifr_+W)OQ(9LZ@Z;rOKPO5T}qiEf;_V>x%xKh52 za_5a)gRjd!RSx}M33u>|gKtIG6tpzmKkz>Q3_Q#!>n6P^Se(I8m-?@{s zNS@Mt^TdXYCU24Vp{*yG2&nTPq6fGrkM37o_aP(7?H25Q#+b(V%fgL&(M;`0mzXtT zWW%h5Gj5I}ZZF>JekS(`>cQX2Gwu^Bf_x*-1k1#KXy3qt|C>zP(U@*FYXFt}B4lk? zF23-8hjI_WHzw0nG}GF#`k*yJ|2w^RVskXNH}i-#_zG`s_?jW zN8jl!jtg$ncY41nJKPhW*STClr>937pTlAh*=Pq_A0_C#y|k9L=b zDgHqp;O>1-M2)eujq%eDEzBAdFkG&{!iikN|E5XsoblBe1Yb3Dg8mx6?8%rc+Yn63 zaPHW}gUY3o?U8JW9>xYyPIT4Yan;rPCTPq)OY{o+mg)mzr}S(qFw8UB9X7T+dK>uQ zY4`{{%~>K_lQDjnu@T@MYJVe=c-B>H5} zV7_lieSH&L=T&X#;F|rl@(uK@5Ybq+-{^%#zSF|B#wH1uLjXtZ>bll<$@OiR_tU22PS>XynziD;w1WG^gQs&Z zJoN4ouJ^33e7j7(%#QbBo9UmkU6SoIa%cQ549pml2T)R4kZ^*?j&0aFkkAcqBd#R+qO&kXp zF`2Ex%lKBnbT-q(RpCc$JllSnvci$~GlS@iw5Bw?)S0T6czp*S$9MG^J7?^l#y|O7 zMpqfX2i#P*%ySPnqhly9J{!I=UehbgnPpLa88`sT$e`#G>97h-dVi?-o9D|=-;EEm zhDo2(_0l)&muL%m=K#y-q`K%?5u0VskO7D2ThXn9{mAQfWIM7awPh(9+cO5`hipgB zOMe>tprK@PI_>C%{dfwO@p{_rhlWcEezx;!rId(k?8 z*5&6lU>jQleMCR`AzPq}-`4r!agBN-*?Y%d(cIWlaML&`<6v=<&Jq1o#>lIF{XY0U z5cUU;5yP-}3Dw*D5DDSO2u%MsqCnbBg0Xw!r86kt6h1 z-^}KFS2g)ybkoGW#8`WV>nV*T`Bo7fJsGO%x3k0Y+0_R#H=&W^Lb+qRa?$(d6A#Gy^_4Gc=0wzk9tCm;yETu7CI$7sIT4(R&(*zzJilGj{`{PV zSr1-A+tYw~W`k+(*mAqfB?`hJ0M=SyAWH?&mifjZ4M9T#X8@k5`LO(froM6Psm~NX zNV)WrpFZc7FFkloeUNU@xf%Edrkvid=KbS?51&*0PUV<47F6acwZ*v{0l50H1;&T; z`NKo>3psumnSGc(6@u#zUREAERdQ6#qndWbmnX)J>X;Y}eCmX+>aWgx#4ji`G1qc& zKQp!-tAFt(>-UJh@+IKwi{hK;=k7&AJ02q%(znhx=Fcf@`0_P6d!&>~AFDn5w6oD& zH(hsZx8&KhW6zlDM=sUQ4Cw8TNlTl8cobYey+V(OX7+hvgy$26JqPbXntlgI)i;!B zYhjh&oihAg)czrHc#JpcvdGtlM(CEHzJ6mFZcim9)Y%sF%1?6Ntkikg9r&a2CC!=E zR<~_reZa&VWGAqNO?8@6@LM1KrZ+P$ofw2~lPswW{-)v*-=?0qCmtc&Qg8hp+1?W< zYxDqR-^{gWuX+ztM}CCLG&y+U74;XncpdA;!6k3VKQ@kY^}caLej8_p_VcXmwm0N2 zdyhT0kovMo8(ll-3en}Wf=N53ZD^CEAJ^Dt(kE%|rQ2kOjEqq>T_2cvROD~GgKxLT z<0`}LnrLkIK_la4jtors2}Xy}zno|)S|MMj+gQlvTr@Xm`HiKUk@`ElFPnbK1Znq?&IcmV1?LcTP8vp$K z=xo6j&a9YqF5gMvchQ^q4hioyo}~X3OOej~G<|KJ`0YETJLL-`c&E9_T<9%(`4>Ni zF8r!3$(er5T(&-%%j(|DV|70SC% zpQ~}xz8F8VBgk)pPxvn9=$+AJ==G*?_Dl!pCjX;-Nzf}fGxUZIDyM!)*3+EXAiZnq zQa(M27>=7Wa;-Vz&(MiQc<17?Dl46Et7HItl?SZ6K0UviXOsEW`NDpkPxD;w8~BsI zD}1(r|JC3xze@Arcsv;Yc6suSA3x1}-`l%t^qlmY&nsa0^CFco{h^Q2g_=9hbb5qm z??m3*d(&TN^dIUY?|i>A7EAwc@aXs&JW7wiqvNg22)|c@M{n?W^bf>i=8wRiqetM+ zB=8xDKhK|+yyHV-;ZM(S-&6BNqZi?od{D*TjPC%yV*VYx_2-xxzgBJ|pPhQhaC}c^ zjrRPuG%iKoVpUsjr!W5e$>x;O2t<2mlRtm5Zft?HMVnx z_C>eMDeu|Q_gG8zyGX_c^G!VMCv^Hb*qqGEudm#v@>)OEedXD>{6DhK*3{=)dAEJv zUb9Y;Lq|!kKg@gS0AIH(dTMxFK`@Ly;oBh2)*86!pr>EBo`tE`=AqerR}_IJV5w0J=BZ`X`J2M$dY1JXu$K-somzYZtl8 zYX3(+jp7CEoK>PTq10!eZ$IcA&Ntb7)z!M&u((iFA5B@xnYFiB@+Guyc4Yjt?&9o~ zvsIca_&AyQp~eR|ROVf^?Vj|aY)NvoH4tbczf$X+iUHKmiRXJ`=aZpERw{EHaychlq>&FOq1gs57$KsQIWrgP_ zoACjiG(L=NB&L<-uk)oDCn--REqm!bQ+u5+wc0(~UOW*V$cHoYLVc?Z8Ys?{P7G*W z%Y1vwjGJ8iEPpqqOB$;QBhyi=7W#mzxz?Jg-p{D>;=3dCb8qzCYWY5#-rl+%cpF{! zFyHS5A9I$siJfRY@5sR|oQc#woArQ3&UPeEqn&=wV(z=Zjz8UJxz^d8TiBo2Kbv)i zM%K{C<}ItgT%I(GpH1^ap$%R7sGlR)gUv-}VB(;#+(ACmZqJ=$Ne4F(bfmx>JZc+n2VkQ=Ak%XbcO5b0{8L;Kzw^A{y&jr7OqpX0dJ*gO@z44zw+ zi=eZ|x1+CiFy5bMd=%%BFXh$o51zHLsxKp!17vtzjVdF zlglE$iC2&FPW|A+#L*f8{ZmeD&cOD@dKGM!Lne4**A12RlGFLNkF&2@^Yvr3zMMY~ z&)SLADt=p^i0q5z#;hw86u(XpXH{PBIQ_!MI{m`S1N2M36nE#IwZ0kME)PLE$lCZa z4vJ+u8-J#uy|*^eGcw+8y+<}Fitn~|m*%m~hs^b}#<-pM@ea=Hf1We@726fP`dKU7 zPCR)B=k!0%8UE^ra+gZ;gy(edya+$$;z8-cz2P|%JfpGV;hL&^%b=RZFT4>GyqmSG%5bEL`e5_v@jT<}n+fwwKI`4w_ici$S@w1m&{zVB}EMQ@kZqza`q*+OYC-;nQ^m~X;& zDy>Qt{zCBXiTusd;+Dd|Jz6UwmNfBON0e4ab<%v3`WJRSQd*alul1p;+`VAV1}5!2 zB79~8gZ3C7s{*@TELvY8THk7EJ-I0_+zhR6&4ksoUSnxGll2BebIOz+OhkTjYpQS| z@6SnMOCD(nOE*J5e!GBI+-UvgjhXOj^($xlm81T>33GPqV?#p^%(`u1{&wD*_HV|Y zzqcja(jWAmX>rmSuU}AoX!|jqr8AkqQU^5eXbIP_CDoO##D^U@mN-frQp25xRb1oz+>`EOXb zrQ2z9mD;rL9-Wqd*uE?FG~|n|L3q7jwsGASgrCs$YSsu=VFO#SfoZ z>ykvc3)$}MPxfxjP3~@t}jKsSC~5u@jjR4`*+FSf1`}y zJ=gQTEj-itU#cHj+mCD}{3_qbC@zfdOAD#{GPbWR)$3&ZC-xf}eJ^LAV>-WucX*rMe?v}9chL-St``p~4TaxI96#D0{?DG=k%r`{PRV_Tfj=H7~CmZ{3%6{H2J0IRr zb_Hdxc4g;h$P10&-t3p17nQw{vTd&Hyi9nhE&EZw?A)mAX3C!L%FfM%H`%fu^2^SN z%05Qfce=83$ZNA@-{+T|9hL2)>>^ioc82q(BYZFL%Qi-3HUF6J$~GcO5_fz?8{9D8=Oeg0Gb z`6J|5)OrrhFXg$$MVW>N!O!dgdHHsIv~$(tjJJcp3%~9g4kquv#V_vXy*$rKGLYJ3X!d)*wm)+21XG?f*>fpg$3!%r!&}Dv-eGX?2mKIvNoNVbb z-=hog|5fm%f7)o{esJSEP3f8N;NBqo_4mMWyyee$d<%5e0N?!?=%@YXEH2M-{S3JD zgUfT^@+`PKBU}z4Pht|^P?I{ZH2^T+O`|E~mQbzdcivs@d{c)GiOu%%|)vG-ljI2+CX z9wpZ+c%Z3rmRK$x`*L&Kze|mMj_c;F-vN)2=A?=fn6+!a?Ty4`bs0{I@~&pT)_3AmqTUFaDvN#d)H>?UXV(xxZ{qoK%kKxjmU$ zTRMpNfYC9O^K&bVE#R3yuU!B*)~DCkXd{Yqd9k1o^un&-#(i)O^yxU=Deucrubc1D@IMNS}q{m#9|)wsfTKFH_s#v`sz${r{Iy_NS_@ z>IM67rsfj0;b>2P8=$>|^Y7X-Yx*6<4*3QnJgqG+S}W8(2GLA8gfqconYTW8{>=z7 z-?j@JXgemu*~rcc*D`FcPyX7C=@qy>r2Y&diS9u*N`0be1kv@igh{;%u{#;X|Otm~q-dMLW zxlvkk^yPbRWrKbEUdJaGo3Ez*`V{b-{hEH(*zK3`8_oEP>!a$v`gHKufi;Gus}k6l z3M@~qYK$etw27HAPex~1-K9B`^kvEAqgZ{_$1}lI-(ustiofVyQ$GM^A9a&;HS)1) z;NoZ|a7w6V}tU1o%Sx-~LK zp9+sXc^RRMEHA{p@H{`a`-{4t(^ z`MPIn*lw?1ikrCc|J}jS%bM}cIBmwSXs%j^zuMOS9{e2UFZ5LAz48r=T+#L-)nN?2 z=jA9iE5Fy}ulc&-ec$}QnYWqt@;iPB4PFafb0_>7P1k&G3}s|P{?KwZZ^Flq>{9J< z?~jj)-t@|5_wEO4{^I=Kz2y-8#eYP2Xk7Sm@8VgqjT$G?!D;NH@o#jEjg*h0=QZ;g zbMYKR#_f~M5C8t;AU5f{)!<|LdhWq}%wv=PqCA{Pr+GHh z^4`o5>%DnURx!JAe7if5Z%_T<i!(WD6jV|TFbF>ZROp>^h}MhGUmCXXQLMXzgKzgIK~{r7Z~rO$xr{V zG8(7Dzd!BgX8AdYC$M)!de-RB#9PbpO68bI&i*%Z5I;!YoIic*>9zf7=W}Tu@Q0Fv zxM3f15cj35i_M-tOeQMx184g``YPri{&iIj;%8su97Nx~{%_?V`Z)M&qRoc(US3~M zB?r;3BfMjIeXR4FyuMdC2hnfS$8(7BB;U*5V?LCshT%#s#jBZvcoli%1K?uX zY&m(Kau9ty{r-Kk+`qBqAl|9+*y6{O^L21@ehT+S_n;ezTR%DLbn*~%ro(-i@XNd- zrakQY?O$+p&_!*Ft8x%iukZD7Reo!c_4_{7@3%e8I=k06|4ww@!u_9e-$$;A$y0o~CA>lJm7hrN%0JV7>2CaJmy`GsIf)(k)HkFG z%31se-uHFL_a-NCeX3AepDmP@Du2-CB(6yn)(YOeHV0uT`Gog+If?6{I%z*A@rJ6L zM8TX3%cM6}mwQ~|r7VU4hwEt;yUbq?B-#(o7@3J(W>2eY&Q+g;tKgexgnJUcZ z{rSpCByXg&5qk3bao{QE(ZIZwe0%llX4|iusegZzlel?k=$Fb#yqfo>>_%dC_eVL2 z$5@<{)38SMq3wHkW^xkAJ1mknxK!=rY&*O!<@mcdVV;|u#Jg-x;>pnNE}N5hay%#T zj|Jlfi`xyjEBW5K zuGg@ZwVJ%YHuC;ff(vl_sUHZJRhBN4H)VlwrNL=J{$jnS?1jQ@mFQ-1Yo+~EL)1Pw ziP!qLDPOUpUhkfq1#Ta5xMj&nTpENY2zEbtfy;t$fv#J@?E-LH3U133i*3mFQ~$^B zpzY-rH_Dr`z*wX@yq_=JD0`xCTTTv+__~ywL}YtdPU1l}C-G3^p_R6mfa7<_NnApE z2a#KNXq1zfVK1J5>Tn1UZ2QBGnvWeo4h zMS0z$!ZV%!j{33K_G57-yp6pjE=Q3Z@UZksl#_^zKPegiD(?lO@0AR6jB*k`;NX(G z@{g+DL7rBwCH%B~Ztl@7E>G)P`@BTC(n9)*o?3xkI+wbp4+|?fiC6h$PmapoNZB)7 z*^?_diI@6i=SO8XQ}*qy?EFej;&Q+2yr}GBl%40w&a31kp5>RF81Wi{Iav7va=|Ah$}lA*|WK7bNsT6QP~#Cj(24nD>;cX{jxKn zvMVV28_tY1^qo=3Nt|NK8Xmz*Tnw&&(5KT@ZIfy1F zaW^@M>#~K?3+QfmtDM9yfuE6C$prfRCY$s4@@bJQ8X1@tpr>rEVv2Tb{-McLw0VoW z$v@mh?%@DAizRXvcaulBOL>IkES8kBNDg61`GW~yk++D=jB*$2BiSx>kgG`Eqsdhy z_tC&6cTstZyUBmtMgHReIg6zsauvzN+eI$k06B{#p^mSw4A^$mc^=QlIulUbCf*3j@!_}oUW zBK}~X`1sF^!$Z*@(S+ z{F#8ykwDfu$W^qtl7>e%Z*e#IlDl?-OFy`fv$)T>iU&ap`ne7G7dU!Ar?a?Lt|C0G zz5kc-F_e0JJXg{48)Vn7XIz+^T%7uzHmFnC~}KFhVqRrFveSMig` zkI7Z^o|{}n?_RlzZwF>|UnPhCd8%y#8o%VOA8IlCq0AoX@z&;y&xC)&dl%a!SMj#s z@nd5-8nxDEf0tavcKRBNi}Y01Vf_3<6Wa!_SWIYyvWj)oET7t+tLWmr+RJM4Gtp5l z?muIq_WAAqZn)--ZY}x`C0DUQX9T58?C*i*S*Mg^f4^(4;_HBC^bt0oBU?LO=kgkS zdcD@Uig%!w;`-wMX0GDf#gmcr8G3wwu3|><5SOd?z^vli^Di+6`)}qdJ`e!M&(+x1 z9Hx_s;J`SI$#8b>YqVp{Reb(2U^8Adx7nYo$QdZi4Mc07U$y*}*D_b}-bY^L^`tf(J`}I=G*zH#(hr(M&hj!I; z*`KQznH~WW57;PS#I)EoI%>sapE<4>?!yKz!ZNbftXUe(eRQw%s-Ks>*(S-*OdiYpz!*J!fTHij~iq5Zjvtv{?j zvs(VhXy;2_ZTHz}T*TrQFb`ozl%?b;8|Ps=dQS#OXY z?_-UjJ(ueJ9_4hedjCe*Vxx`AHj)F!p75dd_MWvueV;<*<_Qk#3#zB{n)EGAt)(Z+ z-&gyS!2!yr^^F(K8|vd)Z9PR>IwuHUuVCt`jJ_K}8zs(OX4Y{yjZd%iBa#v1esL1Ewp&L$$h+Lb%JyQ}Bj{2kt{HOSLhqbaSyUuaF4J^T~M z!PEI)eXChlStDE$y{pIx7Y>h`Gd|uHo$;}f{-vIITd{G&Da9=17xP_8(S%sF&b&4_ z6HnIJn_aoX;KSkUlRtm3*p|$fmMX{LJ>8{_ps+c2-e70XpLK8RPvo`ERq9CQJHWlg z?*AzEC-Wwr!Si#JQ<+QTm1lFIuIFg&j&(=kklR-3n_7w9yDSe?Zjbm%{R0a8Qvd&B z@6F@ns>=NTTV3f;l@QqiVNKFDnhwiECo0JRlMV$EK}8_Qu!C^{QIRhyVnh;J6b&;_ zfHDIkb`q9OP+>qNEh52oMiv)3?&GM{1f)7~)C5Png_z&_^W1Z)&b@W37YsAb?=^qa ztL{DbEYErN^PKH;%BVkk^Dg|f=^q;1NEsL3&OG~S1vK2Y^lbLwpT*rAJK3G5A-)w) z4&T-G&t2Qk=%2O^cl}ZOL!=2mf`QYlYdrP;9PHis#X|@29Nsq}8H?BBB-e*>evRSh zL&68RsDg|1Ic~4vTjKn-zIsg8`%>O!$&cDRfx4(CA*)@D;}y|G=L9y`!zOvZu6luS zBz-LS6(`VGQRl4c+)UjC`k}9a$A+@OT4i`HDf_ycO?S;So|mdKN!#H2c{Q3u{kqF9 zeO78I_%6OS{?i;1^0Az6I$KTWT%5wQ{OynNuYFF;lVtMIFf>ivX{b3pt*t=r1=9yy zd;XlU^c=wLe+y4E!R;AYxb3Uc+2j%)5}YPa;Q4KW+i)6;tx9q1Lv)CW!C1XC@)~7Z z_elNeGg9sTW$H@sU(m*jbv`P69P`>_#rG z&`k3v|FKj|vzp=74YwRZb=6xdVYy1XspMbrq zQ5Uak$fN!>U#slpHHRMDi8a`g_oDV_wJoU4RrqzvnsZT~#J7j=tpA0`gVsr;&q`%| zQ2m@#@+Hnm1;?iErbY5=rm9WFaqOH_($t>r?ec9MOP}l+J^S_}vtve~-O`@_(d?G# zHQJHxOW-u*2dy84Cln*FJ2#=n)5xpZ^7A*k|66{?)6geAA2pzL27M&&v$O8L-T`_2 z+T>R@@IWJadat8z_pI0+WSx(CEOm(ogmczrYP`E~U6}EmgXQSW7+32@(CSi}xNX1_F z8N-0;q0lpHC^>E&V)>g>RTFDuPm5uS;GUp>3viGv} zMsMi8rt5U>NaJ3#{2A-G(mJ?Q+_XUl76_uBUyL@7tEX ze$Dn$-h4})J?i}e!Evdp=Ryao`3KO*`~&{2J^1i5M@U8__Y)Zp<;r1u4eN|q0~4-^ z);-cXhg@f#SM#lFg$MOpeESrA9_>8Vu&uCfUj^pa#|YMKm*lqvh%k$j-gBGg-0g;hC(n*%K+9 zT7}2VUqlzo#=k7)@H2TnhVS|&KX{PwT5%uJ2cq&X85fO2!zhrZmzn(1zvr`0#N-IL{@V0XAQKrflvT&6K^<-BI-Fc}Gk%M{ad2Y3ENlK{6XhFy z?M7W9-tf9-cjMu3pW_6^=DRR9-xWGfNb%xy{PO)-qmPUtchC!*oH4|G0qEE>hAx7S z{dqGQY^K#HNIqKrv!Ls3c=E3Fr9UPLS0n$$k-V3R^a_BX!g>T_sYvDJM_agNHy=7zl z(!T4|pM&TR`*9NKT3;0AtI*att}WJr&F^t%!U&Fkb?J;-rfz2qXbm5%{p7rIWnuj?Fi&K&9>kLeB<8&O>kI9P?}t5p{HS*dl&?>e}g zci$v;Y24l7zRiqgeqS*V<>?9V_))%d*8f!NJFudE%2jajHN{qFSGrW+6c-3}Q>fdn zBG0y^Zz3*m0x~}#jtgKLBo~|wFjZ~qS@lSsUJd`Hb%OU_&g0u>`FU7%PF4ASa4gu- zMTIiBj|wYB8k0+Jk-t%|Nq>o6z#v}#iR6IiJb+mpGME?+t2bNAJ`U` z-{9I#wAq*R6Fg4JDnp;{cs|yT@@~hMKQUKD&da4IKS% zxuYpHR{0%xh+c&Gx?7=Hf?gRsa{U^bTb-E_8I z#65to;Jadh^4E`Wd36Uvv~WUsfHFt>@_c_i&-N|XQoizU(8YW&ao*cVecVL8ZA;&< z2G|?M2)4F(Ki6k`7r`*sw@2NFP`C9Te@D1Z#}%&?P6D3vzN**5@`XS8n$gGB($$iK zqm(x-2VHeO>+YMej=M5?SN5H=j&lQhJ8gO2_@lHj1 z{ApWr4H_qO9FyyPR%yJiESl5bk$E89D`1p1#7v8Rc# z#;J@o?&J=L6SybkCgx|BsE;jW`J;b*FZKz4^aAh~+xV99i9FYQOz z)Y7lL2k#Q^OoMk$jqUM}D?fMw@RMgve&3dR-KYg9~TeQcVj$j7t1ez*l=|tMk?HY2-uby!Zj~mRaW}ewDwr2|rB!8Zxwk*h^VHpL}rc+F*Z3 zT&K!>lrrLP{M6n-z7;Mn-l91+_2WbbkB0>umWg8z|GVxs);x~l+LuPX#EYhe?=RDP zJ9AC&;Y#Y!-C&{Izt+{OJRjk?dIB)PQ))Aw<7g>&fG_8l)#w)LgBg-30i z^S*jsDsM@$5T6ku-j|FjM&)HxI_jr|7?<5kqW=BhdC6_ZdznLIAL+N%Z+;fR~JBGnk8@yzcnoEY+#C{OGYgTzOzmb37my7evLCO6Ylzw zzFiDE=g$f^z_M021fGOV>&nLWl(~)ge*U7bA;&r7I4VQu{xy3rW;?la{v>4jIL?VV zE~oq4*$;dP?OlXSUx-Y<1DQs~pGL-iiHtvkj1MB?)oWYkn~cKaUD$o@XF(6dGP;Z7 z3GGwtK|ZT@#&Lz}wMWedzq|L8rqA1ctdq@S!SO8k0hFrS+Q)sr5MYm(luXt z9_NtDHfpR^LH?%y3;Lv;`6Tcr94q}up2M-P?kA9rFulO{w>tl%dN^&0hVS4Xd9GX- zx1qVIL1b9^L3hK|}zQT!&IAfL8&wt5H$^`7I}P@QM`_RvM&I1HXH z#XgK6?q-~Q}w$z(2rIBexSSf#o2%@)tf%MFXc^+UrBz>=;;mP@VZw@H0bF&V4dpiAdks({_LS^)W7k- z-Y3}Ocw-HAXpw-O`s9-d{<%BDkKoMrLF*(2v86hPm^+|`qla6{$0Fwj59L-*kshqa z4rM=$e`dvVrnm52d36`US-`$+=|jYuqO#`|d>zsr?pdGL+La`%EAge}|8`~`gfHS(f1Big(Ge_YSPq4~`4-YO3teDF5a z4G(^aG}-J*SL_@8llmllt4|Aay`Sej#Alk;$}1o40Li&{n)WoW6+gF>H5X*h&`Ib0 z*fX|dmVd@h9O$0a-VA)P2>1^2-|35DoPu@6%!%_y8&?Vz%?XXakU znG;_~UNdptr2g^bMYm?hlXgJenQ)Y1nMm+ikS2IuEdI-qHStO&od2EuZ_?JUiA&2) z$9Ax_ZXdC8HT#Gitl7NKK4Lc~zeaI>_9eJ^8!X101f@_?Ng zmB+n4xi~8Z*q7Av=0l;^IS-{g3HcFT^sGCZ?HRol>hxvmN2-qWnDFJ2!&JF+8SfWc ztTV?ry(Rjh(F7Y3@oxT$d)As2i+zx;xn9{Jtp}LRZ!nJ>^ZzRHXpH`(n=4`rV(^Es z!~gv@Zl#Ri6Acw-t;t2IUw&*J`$q8`lwvYD#!Y@5jm9;{(^ghH&FU4tGzYBuRsKm} zY1m)rV3n`kQH=Yl7)g=6h|!csZ?4rH=4%}xUtYd*_Ps)iF>Aac{A&%B#+zzuy}v&b z7z^NCV>I(;+oCNR4~RxK{sl()2`c|vd|nsF8|r9{;GKHhZp%IJPC$cNAG(r$ipFK4 zkLaoShER@td3_(TJY^TWIyLXnm*jm<2k*nQfZKR%7tG&h;7#j_gr~xm%WHRWUxi#m zFT&i-oOB;jxDp=P$B;*U_=}~FbA`@fV%VmabWe7J4K9iI-1)eSkA2)rdAk{ldA?(w z!*qb*7MWH&MC;txE3C7u(neEODWbZvW2pJ&3IFAsXqyxPPJqFPagz- zBA;s`!TaNASL47$eSO*T5b7n#c$RLMeK$vjJ~CU1T*xMBEw|(A}yxul*xITo1nachW0|xiw;kb@q0+7@zEp*&OhQ?A)kW zrC`?iVww;6F==|&Pw~K~-TNl>12~@NUE?f$*ZZGT2mRE$^}85tVGZ$O|J~X{ZY`g; zWx_G|^mu!P>Y%RYQf)n>bkh^@IHu4VZCM|=xkGG6fwuIH|1?#5);d^!)j{kA`M38) z&_wSZ7U`R{&uQOo{Y==azEJnqYP!7?*tyog2Mg1SL3-clHs&D4G+8>pZ0X98ZK-f@ zO4gVt;d8_>Uitn{XuiRX5sJIAz70IgEfyLnob5w6YOhd$ll>t@=J+-VFJ z`W~|k@1f6BhyPB>gz($^4Sv$iLB9rpDbX)@Q2eKUy-#hX`l9(m<4^jbvg7T?`Azzf z_MgnJfxqRSNryyHbW*|}=}OwmFu&wGkvHVG3_cWXG&bzuTVWtq-ozb!2en2vX4JUR z#!}Fyz*wsAiD=70r@J%{)S>U_sE!=vEiY-5HCf?%rYwYf>R_vl-2}r3V>jdTun#Fb zNJkbXpc8=U^-b`QZmTlwv-{TI>}xKAk{>??MK?I#JsAH0i*%T`-SWMqp9<(X!L7AP z&^cZ&odLJ%j2JIBrCkQzM0cwjoQNK?9sUx1O0{LYLf-R+QfCij^xgX(4P!m^6&V@W zDKeWT7&@SPt$!JHX&nE8{ZjFt)@0D8@cO&Yzc_yBt*;#2@3=mlC(A#O9zl0%eg8D* z>UBmn7mwT2hr+qocIapG(BS4k@f7`&?C6eB%@t^TxEtw8OWXbG=d#CLIrv{bXM&HK z(8JSLag|tJ0)8&V_RHjD$_Vk2jJkcj4SbWrRp=+HkMfU%tJy{HlZvYp&vWrCzRe`A z5*4Fi&5yv(w&q#mYSqCyea5GC-}=@JJcaslOmP#33zZv9SG-Uj5B=&RUL^khaL=Ji-7vW|I=sb(j5?`WGie>?kCM58Ku zRo1Xx4H{qgPUq*T4$c1#EQx*IMUI|6f1x$ntx9?Gg^?4R8+UX6nrB7NPJM_rm0xfj zN?PbEh%VR88GW73_8BoB$(Dq^*3ROiCcJL3QtCfyCNi8VH=k>e( z86R&6@e=Oc*Z4y*E6*PlXsP(o391i&>8F2`Pe~id>?-^@#eYuao7z!cTh}3Y@YBvF zJxRYh&6T&d))jd7F}lhRtgp}Tu`}t^<<>5Gg1kA_@4ab%r#E<)4ZDo*(zmkL78hR~ zmGi*%Uh-*7@p+Y3-T-da^u%y0o%~hu6$CqDuS+Ore9N~&EA>J{t!21d_0j)uT|$Mv z8ZWaRO?mEcWreSef0fOzJ|-UeB#J-9bq6{;gGYQ>@5{_IJO^c(@ZfX%`T7GMByUrn z$Ms!)kNH91LAFHWheBcOzCwcy$>@{pjh8|5&1fgVqkWCMVX+|WbVMwBHT)wwoXo%I zruEIzA&Q-9Kh7=s4j&nx(l7DuO1>YFK`ZZJgbjYLg7t}xHl`j(VqtY)UT`+a$&mKCe2@i%i*V@(wz1KH3 zU;1K|HJJ!_Fr#eP-Vevwp8Tne@Px+hWpI(;jnF5N{(_gfOHuzX7$-hO-@P7G+CH@V zIKLi#(go`C13Vk9_%2`VTfw_xf)Q`snorV|Z9gvF*W)-TyIE^qzMGZTwJrZgeKva^&IhJ(hz-!(kEfa6 zOJ0}Fx__34sZvfp;giUkkFzNjEBd|AvhweQGs!8ot1P=0^J)M?Un8%2*i}xk*9QL1 z!U=TIc&|o}QR=PtX=L8_ackKx-=9dd9|g7%dBwr6e%0X3>QmUCt|WhnA3UE9z=s;^ zjONcshMMLPe^C#9HId7lwrb}|`3 z``A|1>r1TJ&KLWux~GM;+6D5)Yb*C+-Rql&s!PQ0`Tkpo&qeEs0l(_@gjIu zR@wUxDOZn;W7hV*JbqXGm1E&G@Q^zX-!$J^egihf+J)Z+cwehNmEt~COW2i-6Uu|E zz1KZr;>icqKiRs&(V=}4%GFz<;m@ng6%J2VFfY&=ul*HV=Q}zZ+_d?-hqmAJ73IGu z&d>V#UhZ+~ooxMR>DM~L&E&y0pL83raSv4NBfSG3iF|^U`!MFRw^{Nfky=bQ)<3qvDy8YqG zO`XF(Q<)p$=S_OPHZEW3V!zvx3FZGH?OfM?rEOE572r;=uIRM+3H`UUVmJBE7dwaF zt$wyUINC}7IQafJxaYf%;c*|=V(Ytf#={G3?apTMsE$dlt^#%82b2%aMZ=2Ob`%-w z$mhG}kn*99&Z;`ld5z2SP-(Y#ybrw&MNbG$$eZ$q$-F4E?8xAHZXF}-Wlu8&AIkytI)Zym-t8T$EXW?-&@E<7gq25 z?7lsv8OB>Z#Hk|oJtf8r8L(jgMubK40-;ZNJ^3{r7CvU81ErsD4${y~!yWs;EBZdb zcYdD@zR$!?mEnN$4>=NF#H%^wZsmF#Cu;4jrL`9Gpnp{;X1 z_QrJMOmJvA=7>YfpM_3UXeQrhm3yAi7SV3NJE=Ma>+Y>1x2rt6rStUF`N{K={9p0Q zh2$0Q$`=$*JS2JNYhd$ZaI%K5o?*|lGTn{N0? zyws74Wxktd>Fy53huEy)+{9%kJxD&uy1pe~&)4Vpk^9(g535nE&>NyV;1jcI@`{LQY?$P_j!z0 z=f8@w3sfFmrZ{5=N2q^3BtAgT$*zZUAp|Gbo%;8_ zdR!{Rr|4?7mvt4|L#1^UA2C?#^ch$Z)79wH70vHn9rb^PJVt-;zWzkCV_4z#|XU%Pca4lK+vho3gmz8JGGvkrRdnuog#}qw;&+FjVThelykxy{1 z&4Rm0-aE&U*T(N@d^D+dJb5+tHNCnpDl3NMpAU4;et*uv{+srcv_%g_%j0te8)(!7 zrdLvata;xyj=cCHFG{`VjG`sy}WdRRV=_qjBeTZ9I)v42H;0_nw1&&Gc@T@(0qsXA1?KwjB0jrp%+ z4+=Jt`NSFPx_u0RI<5-RE6z6&?G=kCp{}G$6~hFOX<_X zF^uauXW(!h;QNJZ>JPr#SLU(L^?Zs~UT!pUNDoTtl7sf_`~UPj}lSeIm%34J#8Td_jr zmyMXqv!7=a?K|Kbqb2sbqt!hZc`nA!IxAb>b#Au(lfRk$7hf7rigwi9HLj@co&V;zMa`kyQ1?*ZF}5`7WYzqq`ZE7FiY*Y zHpO!<#y*dL&if5tap!{YtaIZ)a?1Et1pswl9Y9%|h}@=llAe|zVB$ab|OPp z)b@@6zjUDHM}NQi#gT#TS)d-V?`_~=|K(&Hp3%OA0&CC}$0+@-o-V!+e6rOCl5YGt4nK;u z7I7>A{=SRy(se&jJmo@=eb<=xs+ z-lgAKfawsy$NTSzZ+l%M+VU9j(-${i$$}2obXUQKeu-;pHAJji|?C$#YJUe5a z)`}HUY0_&w&z>_69Y?G*zybYDuQS*gxE9f7RCwg(gfAQ71|PbH8;3{fwFtOsxD3(@ zQ-C*zpRiM~o;%gH%8OT&_xvdOl3=#^bm$+cyka!OT^r6t4eE(lFVy{Vjbrcu8FTF< z98>Rr=3P2l$}Oz-IqFTjm%l_F;BV-=bb)Mig*MZDPvc2>XOAPV`l5QPiV?14&pq)2 ze1l{=4rhXyeD@UI&3v@WSHo))xPL?pUG<+x-^K#_;lfixKjkMR?`!aU{!ZYR{eBPs z(8<>8X${o-)i0fgL>%nuV$A1So20my)*ku1S~sY*TQ`wrK0IcF6A8`#O9T> zCZ@YR>c5^eWNE7Ji|^ZGZl2HjL*1%N;~>3DzlLXIg?`3L@<>0M&4RzR7V0vcc?z#I z^D`9ZZN@jiU)!I41u%@pGh4VP=Jy$Q%6NS#T`%1<8=Z_l^-1L2WLR_z)8!kpuFB}E zbopMZod2f0@QZwA`@V_qMfNqh{YRVi&G0EXq8`H`{ZH#6^-cOjIM>)UaVDCRG1FJA z+4@Scx*Fd48vQca1{=51j_C1q@Z$MF>-Fz7JOw^fhRkkD<1&N`nIEZrZ-+H*{Vnx0 z@Ics}U~FfO3cK^4(h z$JvY~e{tH0#7i{a@>9~RTrSs8j)9!z*+3rgq3(WIqc*??=Spq)8|DLbzd$0L_FqMM zWv=)q$UhKm`76@l@3ob=rn6S`*xeyPo`<6?x;Mq(;S6@2cbKT-ylBe-d^bKk3}2q- zZtf^Z$xw(bXyO6rgK|Wb`5n4Piwkz{e#$!HTl!+ zT|T4Wh}xpE)^mtIBI;Z@VcAKq0EQ~?7Cv0K#oC|_-GQQdWHYSX!%_J^R1O|gxrMFE zP7)q!<%H8d;FzVkm(${Y1>f z-L_cEemb*`8H<7Z4BgF%7Z0z;<3@N z*2D}P+&Rn|D|dp6%hvso zYVqm~g{`-5J^$EmZ|ykq*{v31kuQ#%>P(G7TeRhM{>}#U)aj}ly`cTcE>5rlzS^>w zbonbb2S-2Tx9Gd*sCow2WBDEOczymRo|T7wJ*K%o;upv|?d=wAS?zR-^x}6_FXurf z+9S`SN@s73$_&ox?45n_0VfaUiwCK_Dtm3z9>ECkFhggA#C7M`W9`mR*0@P^V$0?$ zhN9=SYQOck6_!S9#HSr+ws%sTerZJh&iR{`cFr%fO_~hf7z~>(?VNwy`=_70DL?6; zLT=KEDr=qLpcx;cgTQ?2;`__YI4{w$21=h!|6dr)Z(8Z=fnABC+(@e zx^brXS^J6WdM2K`75@x^t4G1rAh?*F=t6u@W zq_3xS1e3TksimBTTXX&CeoH@yXW}&h;O)Gq{2Ad5c@QnMFI#;QZ_t+p{K!A8BD308 zzsdI%e(*9Z-(G$6vay^s5Cib>cdQNFX?gaL^zr~a|ESW;_Qv#E%Def|W)Bo=M6U4H z2Uv&oHTxzzDO(|X9q%>LxFRg$+ZOIjH+6B3miZ%+QRLz9Y{%Z&m%%xWnV1R?m6S zmU+s@y0frv%Ku@~W8b2Vhgv!KSMwPItcwfFC2=C#4oy*Tl5YITByFzPY#ltJn&9{oCpbxN)j%TONGEB!P8UOk_>bLUn(*&feVc7qdQa`47E@P>GzevWiF zzLzfdaYq~9K}(JA#B0oDxL8ecOzCN8F??v|>1TAOoaE8yp5hbXS^QqZucMo#Q>K>Q zmp%3s-)p`=G1f%vp{p^b_G-&B%6JpJbaP&j`s3w6__~EO@mK-dF`IiOfFPukGwzv;Z0e#T!VVt(M|L%g5sVt(s@-NWzugDsh+H2qYc-3f>dUX4Ay4cpt(&i)z6 zHQ(4H5c?xMt9*gEj40QBEBiPeWc>7eA$t3*_P#WF@a5zkSlanu<%-S+TcXYf+X^q8 z9?fi<{y7)d5KR{C6vy5)K3+uJ$L#81>dWavvnXG=G{%M6t{#RSyR@_SAm(WY^OM%j z&UIcdowlab(myx{+n1jdDB!$;6Z=y4WLa$&L$t?LE>r;aC2&>XDygzPfxS+4EVk-J3}M-$R?K z(x_vQa%K+Y=1n@cFF$p6yIq>w$_qI zc15{}KPXYhs{1SKbZ}_VI3Vz&FKXp#AVUq%Ebd z8&wAVuJ6LTY``k;y&64z8+z!mpzHw8qFrHTG~fOMczZ56bKcdgd%z}6HQu5v@v`lA zUJ&*E9c3kJg6|YpXXxiw+PMR9Ij795`M zYGuPd<^n${NH3C(Obu&|t7ug#o6)Xp`sIq>lP}SJU$)X!zGm%* zbkI0?)bm~8dz(~B)7T`YvCu&nZJ3OxOUmRb<&&9SG zzyIN%v73C)F|R!qUwfv(8~C7MpBbC_Go6YPlV^7Abj$XeX8Lti8b<(+_wjXZg2pCV zPo=cI-MeH?@F(!&a9?>gy0?36Wor%hT}gc&^4=HFDGA<74b`@1rN?7x4FH9}sc>H}lziZ`&U4gH*pp>q97$O~;B&`X81>_nRC`z6!(& z662jxtn)qHs=%!=W{tjy7)uCITaQ+l^j5WBDkUN@%_yQ#Ovf|>>c$c zcpqAaK8yIa*_g1=u7okF7=Mac+PoIHgFm*cbUI-XdQW&!yht!AX6WqUCm(TVB@5pd z0jF>wUeizgn8dk)iVs^k{5Aai&5ycz^3HVvUa^2Z_@v;uU}8~RXp$UqyI#_t|Wi4z&&0qjFV%Un7rUC z%ICYApT{NoX3LRFjTduaV1w%??6Pd=aYielT(G8R>ASpL9z@TqaX zV#kt4i^1bZRwwzjJ7*2uPd)0x)%+W61*hf*DI=bgpL@H$L38N=^I6*kv*zBZn}1(- z49i(VkGL^^H{U+%>VGf)@~>5XuwDA0xOt_+qpyp7R8xf~wJCTs&U?hIEiwEFCip<# zmkB0#!C>;RkS;$LyJhvLUYk#0%qJKX@9RMKtGwXx{na=96c5*TD~CQ%zPBqM@~yiE z^-X>BzYwoq4`Kk@mM%iqxtMuh(!Z0e7w~mg`M&e6KADX3%sOhjw<(Pf;=z9+B= zj*;}Dw;>t>zMcFD+doFQOb&tTI~I?R^|JSEgbVJhv%4CNmhh zesP(=_BSabyvwGw{26y|-0_bu&id&$?LWGIb#yvUMn8pf^JBn!maqEyo5$y?qNkFy z5BU$h?dchwy?GWmYUZbE-Dc>gPIK?#N5P)pNyleV^!)WT{_jfK?~qSh)A&i{l^hN4 zm%_7aL$X)cZT-JjnCEZSmdeY&(U_+0-voYSUs7L)S4}o4W3mZNL>rS$>Z*F#M8+id z;!V{X%VxmOA+Pij5@JSIQDqJ@RY#ID0TSx&{1jP>hb!z3jRg+nywAt3hSH?()VE-@|Br?S}dLV(S5_SB90;c?#9$j`tErgr^v9fKD|Xc82U%Oo`z3yVi{)qS zY5w$~>6DRQ-(%}gPY>n?wZ1b>H#@>SiOmmk4%{Ptj)gQP+Sn6t=1>Q*Z`%`aOE^Dx z`VeQ62dQr*ZLDH!(X%nv|D1S}v}0%k-dU?> zl`Hsp!D}@yct(9*a7FZBCwT`}FfVv*vw1=0IBWBQXQbu@&lqZBuEFL4U3(4lfqCWw zS!>+FE8K71 zcloZiHJ4dg-C}cvnj175(8XNgtX%)kd2c#bm_K#sC41NBm$K}Q>EH8$n7Y%;;B3^n z7Oh*n61bKyH>0tR;_FtQSJYZhW zC1;H||G0p%e*W!Mf4BhIPw)e@ z3-gKx|7+v_+}z-o*bh(v&IF8obw4ZgXS_}SwaNf@!q>_uSKwRzGY4wF9f;*U2p zkHx&OIm6t&ou>?pY0hvZdLY3^T}hr3|9>hYPfaiX-<)A>9{+!A&Jdc_=m9Tl4puTJ zpGAI5ZQirAvv9nR8=T>AU8ffJ1Y7OL@PCI{%!xF`Y>^ z+T2(KK1Z7y`!MwgKM()b(N=J#=EmMh9{JgdLyUKB?5(c;HPk9d;JRRfBvCfUXTIDq!pudt){j%o94pu(o_yy06m74e)phaD`{b3+J z2PiQYkg^@2ztRv3>&y1NMzY5fZycXJej)QB)2?rBi+4qKUaWbMhxQ!Z4&OJ1Jn}*R z+vh$qXlZjF*don+RDFEg*(7-1)}j2Mvi+v-^0P4zKH=ASHob{#ecwHsoVoi1&2L6l zg z5`7A2AbaKIXpp(GHORQ&dBnxgJRW~D)t$rKm5L{aaty76zg&?xdpmb~{RSDNk0bev zq1@j^oFz)%>spHSlIoG3H656uPkQVqzP5P;U>T`x)j8h0xk6`rz72KH5e?IAWcW>@ z*#|khNw!%$EBku2JDW()>XWUZ;rV3Yj5vnHrI88o+o_~IM0xL9h^Gbnp7<=@ci5k2 zt%1>+XYqR*@3Pxkv-A|}A;L8_)5Z58-PFzt!3FJP;yJ@+co{F!SM5L4I$hc7s-H8Z zeVa2SZ2*5t{Yu!2fDc01%%c5{(rL)R7Y)DA*v}coa0O$?zDC>R<;nL=xRqb0|6`pT zCFFbu&?1u#|IPKjf>?ZmJT!sfFOf#K>=Caq{5k&(bS?#Yi2G#C&uR_qL@4*jTm6V0@$*>jEl!Nw9*1n%-QvV8@!M^TVDgN9W)5wt_B$>j zkMwWf=Nh_oa7^4mEL;oc=(#KIN zx=r|S`fqoh6*ttFS$VVvSZnHjB-nTt&eUE9^e`L2I$G(~95Ef8+p96I-e>Zjqa3!j zp2mCbJhxZL)rXF{pZ=K7!MCEtE$izgn(5F}Yw^5b%|Z%i(ueB9H2Tm3Zz-;% zb)yw@Gd_>4c^y~+UBS;cN%Oj2{TIGMdaV5i#Y!Jy&16*Wi`JDD5Q(qz2EAT6$4L;9kYtW!=*4nCx*ESzRXs$dQ zL+tL}>vG+k7el}5zTHP3FWlLXCzv0}=hq4j=?bkcU)`lO;_+icg*;34{5o9Culv|XsD=2Zr!xeTe8Ww zTmGXyru%!IeHo)Qyu=+v_^yG!1@C3~d~Z@ek?n-PpMuMHggU*QGoQ(D6V&heX?dOg zm>ID*C0cf52fn}V>78+I(0AbRyrMp1o9b(!7^j%N(0#wa_TPTTuj`A~1qMS7I0P5) zNI%zjG{9BnnDD&%eMjOa>M$Rvu8Zn8ocQ+|S104gUdGM;hX#eBbY`dBDa=~)cn?-< zoL0f^yx3KQ!c?$WA?N_!u#QJH*knJs#j(0rR4IU&{)2XvNh%X?Awr^oJ z>EOuL2mR`&F%PQDA*AbW_0eR<^OI!fjei zI#~IE4g<0#3udCf#;Vrp@ZqlDfKfUiueeLdjtCF{z}Lf;-eUk zb(u=|?eyFG5`GVY&EwEdf4BX5zjv3qiuhiG^tp)4;RA1piqY_q^wW5re$R1z{!NP4 zr>f6QV2ovJ*@l-1-?UMnpEci)I&~ipYlcn#>zm-SeI|bI40&FoviwKn)%Tg?%keWG zo`0P=jGmb)Jo+}|_vu;ni#Lrw`JRxK447-{wy9V2H@}r~f@#3d!RTEvCh(=USwA^d z@Kt>rz<=K}{$6VzzDcz2cv*U&#@`t*d7Y@f7ju#M^}FvHk3p05C3UIvoF8u;t?`z0 z7j?*gHXY}DaHT0mWjNwl>r}S#9_MAm7Jc%Z|I2)&o+cUSTZQpL{{1vOZ1gJwN7QW?twHU^W_4=Dk6g9Q&4t1)lVq z>h_!Pb*)b)(1$o~5Z%(4`H0^`=NW9HMb}@v?s38l`BiX9t9rel23c@q-QJnma_Eqks2fK(mDKuzAYi| z9BeUlzLt0E3;DjQJhb~k{KCq?&KB`rJ(wRn^e)oPt}q^B{&%s;>01sRH$Z;X_f1#d zR*e%_M`?Z?=dj>kOby=!*Bdz#fOQ#$rsXWxJcF_4#1@mD%PvlwsNuSNBHJvYgNqs$*U>r%ur2YCk zc1rp6eMuYS^~%q|Eo<8B4AKI;A$&_0OIClxw?)uZ=Q!#8#n;rGFPriD{?USVwDMFDKOFUPaO;!H=_ZlQvJnj?SL)*v)Iu2O3MxMju(c z%%#b<(Hum9f5i(}ySzbhCY5brjo~JEViUZl_GT8N;k(GAb`_JdctdM6d^CMEcz|&> z?FvrW@8Xn4Z;s`sc#G~Aw6QNZH2Fck2mb}0q+Z1b*Pxdq3p(rUlfdv&&eWFdXzY2G zXshvC z2e?g~WrE(kWJ;ZPW!F1|U*&iBP5MWFnf!A){d}?b=d>~M&$ne)1O7P+m`wjE4Sb4! z{?4U^{ByWV3;8EUTAF{Jhd*m+j(`3TraAuED!mW?Ornk~{+U2}Gyd6x9trv9JLo-= zxe@qhCX4|{2g=tPK#v~5-imOX!aCO4I^Tp2nv4!Y_PlIAytC6m^d;0yZ*egy z$yO0wodj&-(m~{xj*{LvAK1*#MZZWdnVh)z<}uPYYI~ge=G*t6Zv-v}>zs?i zG{-aV4AUIXyp1%`>vH~$XYd)8nk+`bm3m&ZS+~1w`+XOI88S3lkSXAZ%g+DrnD~fdt5?YIOP3i+VHmY zYoyimUK@TKKdsBJr7ev8y)TAo@TdH}yGfIbe3SnyUHCBR30(;M-j?1OjK{^(E1XUq z$(D9cuG><{VT7Mj^94(BEJ*1^(j`BN_f?RYTAr@I9NpOFWOydByQ6gD$L=13ZWIrp zbB6yE=k>br6VNKH8$asOLf!Zdmlo>ALrF{P#*Q$}@lbo1=6Fcscj-pOMzVC{D@af1 zM)44|m9Hz^xTohMryJku+89YUzC?V~RgYuW;%Df2*$a&WN1XGZY-o=KzjR?F{RtdN zdz_v3z^uGY^tHw)evW#+;Y)fK-Uil0thQ8-o9KM#RTej4-U{1WrEeC0*~9(sqH(p> z_)`ydPkLN;E@-@~{%Gub67M6$JOqP{pQu~qcS43$HWAYp!3KESA%84UCcr~D*WlsQ zyf|Ys#^TsC`l*;`eCBarw@Q+Af3A+kCGlC#@wqwNAF1-h9&|3Pt(_HqSg$I4Xphse zd>^14jp3Q&SX=o;R2DC4FWf-8@|-<1fPcT3wCd_OmZW%q%1Y;+rgW7PZoBvn-VZ0O z5$52$mVkbW{U|o2b`DfsOQ-`Fg%A0%TEA)ai&iDR0k`Coyuo5d{Tb}cnN-)?a=zI1npCGhraHF5!y({ldo0%v@4yK z=E<8smY%PfK2#Nr1S2@t{G#lzpSRUGxWG8r%Zlb`#234$4f-SfA^lgt=Y=kMrewFN(;E3n0IG4%Ak0oc47fUAwq4P>w+M@F7&}HhQz9;Y))Srca;w{)sV7z_u*5H26%hdbO|h@o%Pm(YCui>R)8^jQKsB-+8sG&)3O# zp=@_KP-da~_OxP!;g|$iv=37;Il-kpli_*t+S_?#F?)Prc*?LXXIdSB4#~iA7ELu* z8r#&djVnBV{*|>ClquEy!w4R%`GhWAI>KW8;77b#``*>apU`7i@||VtN92F$2-;R0 zNV-CDVDiBndngZK9p0}}xsF_rF9mmzxqACva&pxtxLe>k={4?uYLEIQ&(gipW3es` z@Dt?saxWg&-aqk}d=1T!!h7Z)Xg)xCQSqw;uRFM!`R87rn;uKyc})Fo0;kC(aAxgQ zeCXrI3GnBDcS@Qsx*F*#+9qsSU^itK{aj>mm*n}T-@U5&+NKw@S8-$h-p-@PcE0Hc zkjDhf8U1=^I#1Xyc*DmBM0f9R;*%TwGyTn=d`P<;>8CxgE#9Bh%uo9n-_w5D-hw&h zr|EphEI;ixN>BM|{c00BCw!FPk6QQF;LQGO}>V54IFHg=!nFWP4)1$Z+$O&|Kd^_tnbZyCPscy!+j*%wy+ zXmh>yV(tr@Ul?8I9h#9Rt*guid;c`$n~sz>(izUnmK_n=?yR`Lj#~R)^?OH%3fFXzjd9_#q<+= zn~$`_^*O!H=#N)suQQVWD_NC2`_FHSK7oIhqNyJz%9lNQr?H>yCqG2KS9D%f-p4-= zMCH;qWd})Ln6q_*@@;3lH|k}c_oTuTx%s2{7V?Srlij7(&K}V&F@9e+T{pSL_zS#A5CC4&eCM45OJnrNhT5SD@dxmbi zO^kKNpUq1td@CNSc~gxySyyN=4Lwiw&nbF+X?eCz)L6vx9c!3IrL)B!7~g8VtluN< z{4?>Ybmf6Rqwnr{u6x#e(d>osIuCp1U2_b|e>iz$GoN%i{UF{IOG)VF6kUx!Qe)fD zFYx@OdK+Szp&fQ(GQ$DzXiO%)webpN2baV!%jf@yPY16)9;_Ij^3sp%6Y~TqJ_+H` zUXm!iC%La4zlz62x@#4>Fi!aZ`8B_g16N=EmG}zVN9@TyU9qXpQ;*G$W4jcW zqyFA={W@H}-_Cczu;~}TK)Z@%*g1{lb8)g>)<4G2GjumUU{xoInu2jk_^lLBa zuh}#4J9)nI#P*w><_BKQ4^VvXM#`;sd(5>qu)96#p9}7YS4_1wNh`QMK0-WBY5ED5 z8iPv@>s`DM#^N-ORpTvqq|%PQ#g4>l?Nhcm;8W=awWs*yZ2m3IIPJ6h*6fzct93t*H4AU*yB_&v;UCK*7+ z2Pd@LIGkGJoxm66WwScye_Jd4Y-=s=$=o_ReoN&h?MDdwfS`X`=P4bc_N61gUb>C*Janh5hH>@X&;|m9}5At=p+yDQ3c5jky!ngFY@fh-!z)v%p z^(AG?+k+o%cI(q~(UvuCj==m#V!`kEZtPRazkD1p}z)+Rofcn$fODqhn!tu2xNC-`Kml&_-|`t#5DZcBm(OF>)^ zd?}u7ekk~>;4A5zkRkj`+0?s%TR4&4*7_srDo-I6p>~v4^S=ebgO9T>y z_jY0Z9(i}?yOjgpOIw&13;iVrhsiAEPO*0J3&@*8wlu$Gwu5)+!#D6-<3A?!iW=eAbx{z4}Ik zezh|nT2f~|>{dVXVRuJ6>BWdLFA+X~Me+K{Jj<4+&wS8267h;~W@j|O`=;B79ccbI z*A|tv)?4m>FISSytqZpa4kbh_AC@2jb*uXDc^>u;Oaqj*9 zjr57=!@A6$@N9GXj9*OV@ag>;k|<;E|37%VgH!EyI0T&fawfmjV`B~I*KF_q?`V@b zP22mwiD&gqysa_E%Upe0GpcWiHNwc>>vMnRI=BoQ=jZVKJbXN#knIP1TdFeFZ}8~t z*Lr6oY&U(y4}a46a!V7;-EFZKh%ln?j0T9zxwi+9f9(_=2_ka5NL(W%LOGD&~f z`&Qz)2AZb&?dR?lD-i!kM@qMP-^BBs^h#nMF}_Ar#BR_Z)0On$2G{3we7{9~&fvWl zUkO`9vBMTRHIf*U`v6(YvhO zx=(Z&WxYL;PxMy4XZb|`L_YaM|ETh`6Z%9tb|L_b!2$jwMT z(RY2l;OqJAIefiFUi3OpcGBxXi#J={qw7M+kT3UA+DrJjUD@NT18cAc_L#grUSe}g z@VWI_I>Yt7wg#y+*1ePR_+*z)x&@zXMdJL9R9z-F$Uwv#=*(PveoIFYebeREAC-c5 z!HaNy*bjcRV|>Zp)jQUgR2+6ZzU1}vL$t{BC85P}oF6t>|Jx?t!Wxg-`C&hG<-9NX zBjE#CkuUjup0j+({}61nHIgs6nLIVWu`Tu`e;KANjD5)gS6AAXyv*8;W%(KMjK!CH zQgxYZ1iTySZG3q70UD!CBX4+q*c0el#x(Jo^(DwR;J9j`b4ZQgIBIk z=SlH!I1Uvon^+$pJj)k*CRjJ%i*<^mydXcTMv=O`B(yC+1y zjh4-{`;30r7yCQQwr{^DT2?53{ZHF}%X;kc#1wyd9j1F|w=I3RXT9$E>6_U9{6+h$ z*|v0F*8WZ`SKFiEPf+*l2N&8rkIoR#-B`DE?KN~e_gm>ss+C=P4y`Kfxo-8`J?Hi8 zk?)_)yxE$*J=XQ)rw+0HgL5*b4s}n<+g(+EON?yeEY=t17c<6O%sfY~|5ok-kvuIz zmll>E!FiML;e{G!|L};)tYt@5jz@=_K-}(6InLY1C&C9a-Hh)x$o$(H&gAIn;+H>- zb2PZ0=wx(7*JH>JXucf^D{+9l?Bgd+vlFye1Nc$k~ zE4g2)JKtjWTz%{V=o-e<)q{xv(1+?A+C7+Y_MFaPoz?NuOI^&aowN(-5B=CkT~G1t zU$^Drx_O>Oz8>oIu|R_x-%qhe$=-VM#`46VRDka+(>uBTSMjcx zz}3*C08NA|#lOyVXY$+yj77A{3#Pxz3UA%_A8QjVf!2LVw5ZrSdo<-+YkF#UJVLlqp8~$Tj%6`wI z&M-!3Iv-u@=Zt+UsB#WxLQwu5w+Bve_?X2;aQhUx>|J~d#}`?Cw)BYEsma81r+U9D z@UOeFePtFQH2ND&@~#<2-bcof_w+r-4#N+#^7ie7oF9OkPe;yoDsSd2^UdwZO{sJK zTJX09JYEBjT?3C@1CMxwUt*K`GW8gvEqshtY&zQgH?*}RXW zE4snqwxwsIE6ze!?DT?k#c$}l=?V{bhOYRy;FhlNaEH1AxCg02aL*5P#UEY0l_t94 zQ0m;CSKM`{HQV~2C%RPee6K53v3_JVXI0(C{>-Pb7r(@AJVQLR@>z5T?O!kr`9bc5 zPp>;x=g%IxjddNj=g%7YCAQ@mY|9`vU{gLSzb+RI??IVuOOIVcyYb$$zPi5~`m0{2 z-1@g;*D1@C7af@J5leacw=?Gy=WBQF^~~k$WBSrvnz62XNz}g*8EuPl<#&Dy90F@W z>n&^d*-f8!5jfS_81oNi&MIrYjlYjk^E`I0m*MV!A)n`J@>De@GTi9QtNn*$ACcK! zkU~tRY)TwsIAbM*6o#k-lt8V!3{m;SQUD@_$?<`*F ztMj60p}3d$TXC<07Q`@W9M;_)^}m((*oSUz>%y_iHRMn5WvVT;bFJE-O|_>sjvr?m zed#v38vFi*+0l)%W%s4?rr`18MUC6%A5~vla}dvx(}un`E8j|7Hz>bvv+^leL)_iY z9{3l!wkE8njkXgsCDHpDHF%%JBPn@wCfvv-OO`X zk01Zwhm!8cKXrINiVRwP?bu^r#s=ItAo#(ZaHBrIo;3COR6FY+kf(fLo5*jrsbu>4 zi8jV!e|ijk4NM7r{p|F4e+Mk||A~nH!#HvK-LaFiLM-@AIqw z*t}bLmycvRXCwPP`l9k0($=@S^@ybXi}J`WqDvq0#nZ_`Et3=2WNK9oF3gaa~I3Mg#5ci+g1kozsWPaY-<*JUAl+qG2VaT^)Kyeo-)?I z2dsOXZ{6VRad5>xprlR(S3S=LI+e15U-hcY*C-R(YMr|{^N-DS>ILGHH1Fo?G?d>NPxjygkmdjoqNHjqT>dXF>fd>BPaNhJDvw?}1^Btzw;_Ins{S`npZU z!|fHbU=;zema!!hgfDBj>d`q5u8qIvwo`@FuDlVLCYuiY5 zh4cLub8a^|+D81$_Q=3H+d>-<#s$o80tU`GHQNJjp2xO4KVp8@zn6f!D0<7SJ)ir@ zhMxJ=t-0JgZjJUnZ#uMBtYTYtd(5ob$Nah4NQ;I=aFzG`?bNMZ-_EV~4w<{x)pmJn(P2 zpLd--v64F@SLM$nAMv2}&f!wSQfqhaM8Z13#tu=2%8J(u7ogkvx2ne<#7vqhL~<`fU9T=J1|T+zS56?dbD0CMW0l{_wu$^L(5UXTFS|=hK|2a2)HD z@T#pOO0n&QtUeF0X6Iqp@7;nr*}|yu2#D zwGqGlDRh{ZSMA&R1?xbCU-aj1XD=lFiZ4+_HnayvJg4;_UI!h*v)WsTZG|~HMwfD-PESQYz zLH_>s59q!oeBycetq!&X|EF-JHr0-FUjnCj;VVyEFe;0N;k@Zn(CumDRW>hN!}}C< zJ&jCgy;3;W-z)}m{}gaX{zSVe_`GfyPhRh*6dQOTsh?E$eZY`tKZR2p7t_~!9eulJ z#eDngKfxowF5IzK%>1GsP?vZ>^=h5_H>~bJkEH7MYvx3+G3$oF#)oz_j@xp^9yWdz zHhwiW{x)p<)3c=OIa7}|bhchGhpuN_TPSw+R!f@OJCpgosQhmD$@6FrHh%*7siHTfCqYWL$%t`!@heQs8k_p_;IB%A+zz9(#cAOnKqjUEnoTs9#+ zzqRc5;(PICe3lnDxX1ez!hvj{_Nm!_7`=l>|yte8v{I%q$o?16kb_D}XQ=C8rKGwY0PEDRr-54e0nLWgzm zUE^1cxowa574cob9RtyPN4v~EzD<3kTsSV?o>zQ*GJ2w=obF%fYudSv(4!mPl8zmq zkFQ+;oa!5WSNc)BUjuzB?as$*?;JjnG|^7{C3r2~y*lb=p7e>f>lTzhz<1+4aI=y) z_$s~~N*zj5%zP=|yWw%Q*~YWUXFGR4zz08n4|T(5d%FBuqn%g#;MnTq{T#g`J9?Lo zC>yIe?H1%dzYp>;&^dns{P6~2SH8^tlo<(+dzcIy&m4%4RqyBGHT$j;%x?rv#U&Ka zkK;J|t`lq%9h}|yGlsHcqxZjZH$H#S*O-sYVdvNvnxE94q{;N5%6<5{cSpld-UT1s zM_&$$hAVe<(m!Z&INv@Ajt-}v3*z*j@jgp9C^?*zI){HnT4iqhu6cv%ouXe$jV8N! znk-;EaHr18Zs~sn*o-DT@5T4Z-Eo^=ReREzt0$Bz2S>w~i6#d_OY-Y}4wb3Sp^Y1W zhdj@bUOf!>uH_pvsN^S<|JLE9LV96u@?L>{tHeBA{Y8HMld20nXm!nE9aRn;coBJ| z12-Z^7r|Go!>DdVj=H9;>&~Bw9yxXB>ni7Xu?jD0{NqQ%a7x^n#w<@e=&Ox>fs?@+O``=mAb{nizWtRE@y zA4T`f?mx7=hIt};zIaY~4R#?~84=Syaj4tXud}AtQr9BVZG5q@wY&n|dg1L0hc`bI z4NvauIcalOu6O&#bG?jn@40Z`73Iy}jE2ec#D(WgEN^~18s7BX2>rP`Ye;ryj{78y z`|c_17UR2zdE&Lqb+2JPOAmA1_U)LX!22G=)feMOXf4ahLsvSS2)D!ordA#RCt7!L zQLa~W+xWL9J^MTGzf0%vVZdH_0K8qauvaoFKk6{ajmtOa@*V2>D16!(>EL^=%0h#C zMN4QkhkrNEo2mO=4qV7nS-!&a7k%B<`BYXr*cJdAe0{EDMKA~HGgI&Tq~66ZyVADK zR}RbJ<4;w;eZPbc)7jHHdk0m&+N3|fpWyXQOZiX!nSOnkI#ot}8v+mUGjfw}w5fOT z-TUJ+Cq&mpPaRs{Yu?r9GpUZ(Odh-M=KZnTfhpl@L8qEe?tDzU>l45Hwr39@HhtE- ztKFJHox%HdY;cr|=l}G+;A-jqWhV_$FEZDwm~A0iw!3Vj`YXH0d}S|j{(GYC8O-hA zui5=xo%4HITs`{MS^p;8FaK3>G~!&>Maw$0_Uj(4{pz8ebX^^qql}{YYESSkYH@R{ zsd_ah;m<61soPg$bp>tf-WI=RpMGqxbxi69vZ=Mn@;9yw_Ap6i3zuE6PUj?44`x2O zWyt16^^6S1{SNAXi~1#9yGv)U{F>^FWxe^!TW-0_@rvP0b!xxy9@q%YDNt8W6xWMh z?!Cn24aYHA{{1Y@BLgp6D(U^zP59{;U>oaLz{WNB-U(hR#eR7hmyf_#Eq>BhkNqUZ z{qQpPE!rATjfJ%CT6k^=_Q%T4QoT9S55>-FuUD9tvEgEqFJQ7*3i-*q;!Nig=-a4c zuw3Wx25^w*QwmpVlemVRss9&szETRe<2`p~+IzG2>e$!|y)IdNx%h+rC-9hRceHb7 z?A#XWiomtT?V_uEfoA8a~7i# zP0=|ybQ8A1`l$Xc;5pG(*G{u~HI~%=64kHrPg3uqCUuyM6yv$8gpA}H$NV*%wgTra zvCpGXEgxGrH=P4KyYMXhX^)`fO!aiMM$ojrp8Cd% z;DrFEwf=S`?U2W}=%YD`gxt9DqFJqcR-Ju~Izc#;k1t8yh8)lvVCfyLqsjAdop!?TgJ zrdl|S{JG(@=jB7|1Wu(r!x?S=!1Y^vVCO7hf91~;1Ek)@`^MGw_b4-w{Cd`WZWfIm*^$_%fW}COR1&`!SG*$i5zw%iM!C3~nC(!HXnQbxN{w_?T9*w%RaM5EgG6yJ!)Mb~M- z8t%#4j2-p<#lVtyOuvCNAKMblQBL@-+nq1HTRJe_lNV|Ha^t?`iSo@d@Dtm`V6UL- zQxq+`z6NWC{_%69;uYo`(d|XML)-2~2A3VX#X6}cJYNT%GM^hjJZA8npoxQh!b+!8{%*3 zlWx{?8gA*hn(T$J!ZT*~Wy`SxnkSL`)Y^Fgch*;)kK^vLS)mTO?OoaWLi6&(w37Ve z@NMIfDH-FDzHB*?-XE)s#AB;*oH4<;B`UHek^XtxKj8YB-amMO`l>#Fcj81_-=yg{G5o(yrE zIY0bD%B1vyw^jX$YRa@fr#+AUn;-kP`LkEaV?G$TH~*M-@l=s=$~%+# za{SEC#-F)n>MKR$_da^MRG zFnivzv5{njy3TjLR>z~UU!wXYJJtqsoT67kH>K)J>$6N;hkOU0$#(kjnZ^&u@o@LC zv0ohfSik^26vr~(0C++>ljW024{PmLgU_CVSG;RTsFX6S6x?xy4Cs!dg3d27r@e>vPLhrcJ;^w;8=Pxjg00hJJbMn`Y(3(FTv>Ag;up&U-rmi- z#>3L5{~vqj9$!~g=KXzgn)aknp|;4KwnS-4)heL0h&JVbl#8NJxkFnD3gQhJuhp?N z^ST zzUJ1Uy?$F=OVZj%#9C$*7uoZ<=~TwCh;}u`;dsrofR||nH{ZjlG*<V7J#uHy}x19BmMjIPaPBeZF2+R(A~zTIs8eVn^* zL!?x7GGn<8Oz$06pO6pr-N3T*F_!v%XI=5qwCOi4R5Vz1*|$bjl`$ z|18(AZ)5b0MQooRIB|aeXMGjAYU4mqctpJc`dzvDs;FxUy0DUYihSPJb%y zY&HUP*Z4ILkneNVjA`R4`93~BFPavWR7UuT>0X^vz0Au7Fga)Ku#(dSZkwN~%Cu;G zIeiqSHI)?4WDXB>-RO9EEpXJxmZuTzSM)^&&ZXmDloczgQ`WCNk4@96C(z5_)8?Y) zXh?WEjP_;wTdp5CepqoebgJoIbnGt_TXQ}&Z%NBcI&vAW|o*|A4&4_I7=F$>0BsV99U*z~NJvFrj{4`cju z8itSUnXaSWKU%r^ddByyJ`@YD;GXeb3Qm1H@1h;?jpjYA&HNhp61qLb1GHl{dOvm4 z_65Vhk+$W8)8+8_v}NN<7m#Pr=k5(KTZ^{72b|JB%1>Ow|Ek^z=8GaUgGS4gQ$afw zzSeue23+!2rbbgA^*YLa_lzt@$eqc1)R-&1nBO(>l+i}$Pnb=`7>C23a3DCRRNAjk zLq9M3x%@?&t9Q{vjhs){xa{?sv!^b{$K-FE&Sar|y=&T^zx}X9bKE-<>7Jtihi%|W zdWmwf5zFwVa3*|++OlyVq0Zm+cCYQOO%~AFf@}bt) z1y_RRqlC?DV#9YfY};xReWo(jfX~~&9XwZWPLI;_bO!8xyo1D%YQ{RkyndkcA7f1O zXgII?b@iC;!_Vsn(Vp1=$kQt}GZ$pQo5crJ-VSg*in^hmDpzD__%=MgG}lyi;edAazE=OFa7a1RJMw|ZF?^}7k8b(pCU6LkgnC6Z zmOdai?cplk0-nmio2|!Frv-VE{*u0H$?;!(@4!0;M-$ug^tZX5{%hiA8Stx)>~PJ^ z#KN{IZz65?|N0#}CwJm<=r)ob|5&k2n5q_%Z(_^u>_V`-yorg-;Jhm6z+& zY1pzoRr^)8o7Sraq|@A5Zwok5ZcNVSjzRB8@05q2cslV$%YWv*;*$lpK4tcXFSkcr zPW-Mv$>VTi^R`|DO=2Zt=?)7ZErX|0Y{1lr5L!8}xLG9YzHp?S;Fgi|)=`$RB z-lh_qUXLpF8^YxMBEdhB?YiCbPil|r8{WQi=ASbBmcU1b{QYtiWn^1ej6BVA#n{fT z;K@A;cldC2*J;+b%DW7BM(fvjo-LpQy}crt$Z-!=18cv_nJXlxBaOdqf5BKP@b)D2 zjQNP@86Q(GG;;T(x1GR2#Pi0h!*%Omu6FHvewXeqp!>03%}-Zbir?vd=G_h{+q%bMNRAI0g-_?#ehkOpd2Q;B(9@s7HlnSFZ`wz$-^1l`{r?)%*_?zXCb=Ym`GAVxX@ji>M<|BhgI!g4*j7tUeWlso4if35di+YLq zA}H(e8uI3N#;(2+bwqdIkiHs@_8#tt_>lURjO(}f({wDkL@^)6`vs&+T2zj92eht6 z`IcOi?7OYjMiwlYF}FHpt$+F$PPckk)o!Ak#ttq8TV?FQJIx!F_jUTuv=@UmD)qD8 zZA+UwpsZp>(8bm`GvUtg*}Se&tf14~bt@j!b1v7adl_xL(Vm$nY|Vl1TK7`yNI4B< zXsYkBKM&S>a4!8Me_4E|{0Tj4ok1}S#kXg;cj3DCiNrRt;th8a3y-j$vuNvSU!g+^ zk2dE4gZL;>KUGdVa;(MN13R~;ItP1sOY6wDju59;yCeBnDpS`sNWtmnUdd)AjZ@|S z+uBayZd>%+0HbgXm0`@GO#*oP}@`XKOq;i!;q+}}CeGe?a#kz#@Fk}stIcNHm1NfygL<-4&*51) zS$*VU{R_V(uR0%Zk=_GmiT{fGUkJSZ3>*0hd|#mcn+z}2b3&A3VEGxGQ!3b&Q(xu$ z`7m0G_I8D2^6n?_x)vrdQra%w$RQDW; zCVu~!KU4k*#_}Y&X&b=LM(8G8?)Rp}=L>O;7U%CgK`zCUiRE&eBuaff3HqJF)CzBlXXRHvBG*N}s2_ z1kK=sthJ|-VB7}!c8vS`S$Y>axtolQExyk&Z?o?x>ECDITXueeCgI$b5Axe~OX}R0 zF5bcUZR_*jXgzv#`E3uN8xk;QjB8E0Oi?*g1^2NVX1@uHpY1@U5^xD(C9?yCrVnOs9>_ z+$*8;chC{i2?yu!w;Rz}R|aS9(Z-gg^SKwm<*1@_Oowt01F~y5?aJY@GxvBWSiJsM z|3`rX)3NBF#p|=z)WZ0}C4$>@QlKMKxb?F9{xshNI&&5L*Gd2Ksgx^SITwYw>`8Rz z24LQZ?$lmT)1AyC@*Dm6gTm93;AI1FZ$x))qAjmGJ)U&VNkVs~;M6$ebM?M7LznD; zvOYI&N}A6y+WrwRB>1%_X;T^vfc0j_FBV&3{`#5ApAcRBIekylmUv3zvph0uJD^vp z?J&2={K+(Zi&-(0G29Kb8)tFtj=YTF?&w^v?E2NvbrrO@m^uGqY?O<0twXnQW?j^T zj5J29okv67KDn9moNnQ4r(3y~Yju8a?3ldz6IfHO@ zet*tz<6J*rm+iAD-?(~{vwt?_$F1H3Z|Ka=@=2Wc=U`qxx0SN7?X!tL;KG)r^<)a|1hTIZMKRJ8R)^Z{_|%rXm9chwrK8WIsvyjo@NiWlXuxE6IC)VkZ8@bmqzC z*cKT$9NGGDG)L!5${y)Nj&#Nq=Tdg*j4+Me&eGI*SLif<_U9aI7QyT6mOEzTcCFYg zt9s7rYT6^}9+(MVqYo9^eXGhdMwQ3zC%2XNxm@Wv_A*G9a&}>N%Nc^jow<2Q&d%Jd zrw1=bPst7m_t|Xik>C3w_7vqyVLk0X3;kR>a}yj?&fEki7jaH>Kk)X=ir-(r^X>4A zKhLr6(AJ@U&{@*p&G?Wp_%k>AIR{N=ZhnF9#&5IWVdfm2x%n2_&^zV5f0^(8%*~_u zt+}>+B{}Zcgt;?m8=GymFAp7*SAT>n*SC^=Oz`~g&!8M~@k(Eg`uhn^_%-e)*ZW~T zLrc}!O~0Y1zG?51_|kO4^wy#L9ydXsG;0_znA@@A+2RVBc7gIfh)7{vxp<=E0vy@&T!(oWyuuJ6O?`w`&e`?Y!x&#I3ye1|L# zsmp0j(0`W}denxON*mAURFXg$zFS?Jxjva$49hgX$TuIIp=e1ix2nj1@7ea;-; z3a2y29-HefypKF|ox!NMYq7Hmd_OeTUA`8X(0QSU&g%}(3;kEYIMcz1 zjFoQ$#v5CQex>)wp}`J6>MYap)%=H-`<676Bp3aRq zQD=4TqB`LFH1I0kyFok;ol4>lX#FDnhYxn;|8TZ(9yqracP+^us+>i-8J<#p@{i-Q zNMDa^9icG1pT1W(`LzkT0~{O4=W zHT?kJy=}?&XQBnKtnR^S%Z{7nra_y$=G$PMqfS}xzrGjP66Jf6_dj>yClK?vs(Kg8 z0f~89_KM=sp}*Hz+`FqCzisTL!ojkW*U$%d_8Jb$3uUA1&E3#vbX^z?+8J}ilHt8G z2EUGNru(Gz``h}>U9%nB$-II)oZFW)^#0JL`Ap>q%E; z>upi@)>C4=HE@b#^z8V45ze5(PwB>2T_F6IZ{xi5;u&4Ut}ia%dgDI&X75jG?Vj)M zxRfoTHgsNT|E1`h8;M~9&!5#V`pU|~XEPWpK5h5q4bk03Ti{grttyAU)VbkS7T)vc zwd;&DJL8@?4If~3x!5qLTs%s;>}2M@7#4h~vfINoO+0lL{3m(2J?iGnRp#Y0z~|Y6 z1Gg>fUX6b|v4in-o(v_B4e z#rBkO7iq_&T?$3*FRDS;Z{0cTjf2(Sy=e4W;90OY}ZX&)`*S`J%xw{1;5a z>6{J7Qq7&%0S@KUAME;hm4}(Wj&yzLj1%=m>?xMnvyfTzLfS9C{MYiOs<9o}LGp#< zk3{J5S5x2Y3G~Q;YEw3ETPpXrUj0Vwj-bBqHd6f#+Fu6@f>r*X^o)E^?YnQGe$9GF zTeey^y1dHF{1J-yz)w3yd!;(7MGz>}wfYxyzoedaxScL9TNY2RI}`v-yx zp0PF<+ex(1kN%VYqFlPXY?1Y`4}701D}U(6{1(sX+l!jA@6jvpp0&-%;e7#`Gjcw$zj`MI=QcuL+4^DI;X2%OcwatU94`s)cMx2)cciE-<-51QZ)#gOpV1_|H*G%3 z>j`iu|9>5L{u%HXU8VsmQaQgnh+RgR=@1*hgS=^(|U1cPwd z0p4yAK3!g`-u;wk#&7W4#&79q@w~6|DDODmd(VW7v}Nn)pN-9+ZJn~p`xUNa0yAUdhvr&YAPH zS1#$!YCE&tVuqXr?{;WzYl^O3Ci~%q*Fl3QicU__OFSRnN0!#L(tZ8nf8(8>U^=uXn&acQS{EH@ZhIfHTAkafcawR)NxUJus}h?J@MrscCXRdt zi1gSl;zMLp_Nd~=rbB3Zo9;#JX=F`p57yLPLhQ4Vd&#)Vk~QZQtg&bwl1<+Xp5$xf zSg+FA8`clDW+!ps7PAfZ9b9wizO%Jf*3uN+WqJa;UvbQ1zlr@MU1H_;q3tsVUq&5` zvy-*B&bBiLFB})w(eDCy(R;O_=O+0s_!Dd0UIzJZ2T*?%?YHo)DdKDbe1JvH2GIAH z@Lhek(6_D2MTN~}?o9B$f_w-0DC(npJN{JO-7i61`2q4-gu{MsR{*=-_no4-Ic{Gt zn2tO}vAT0~pVc+w7@6Pr9oqHq`|t?U#W9=Tt{~lb+UGAG2b>~h0K$kMeY^?heg_8 z?cd?{%t7Ub`S^SJP~Z?>;lp?0w_on?QfAyX*HBmfy}qejxfvYMrs9ix@y+F9^ItrDsqleqCw%o2*AyP#s=m-~dZ)Obbd=!HGkcv`n|9-pZK=M{ zgX^dx+l_d=?k=(N>v^ZLk*jNE(Jy-M$12&plV_`ku4w^Z;#2h}dF(_F^zmJNYmK~p zWrOVl*}yyLZROaln4ar>nrE$bY29qg(o>1Ky^XkkqpeA|b5GPSejdb`=qIf!O@}V# zKCWW&oZ0?ytTm2X)?Im@nnN^iK2G0H@Ryi7oDK%}c0NFXap@eCXhPE!m9|_N>$|ZI zOv?k9=v%n<{(`fWyX9{nU){*p#k$J{zb!s1Hv`st>%eI+U?){zYm+bA#2sbCfi!Pn z^C>^Db4k=2=PT$AKgGxv4&EW$qvs3IOEJU(ae4i9Ub_Em>3pptXsuf@MZvFiJ3G@3 z9&4bS;$d0`^5-?(HQxr@9?iQSlTRZ4)7+)+vg>8v zX)X81L7iXoTRc^m8qKl)0i2ze+C4^xSK_JkA-T2q6Y;6J)HR=v_sTg?KZ!dMo5@28 zc&r{j+4~et!`wjei+D=B{So9h!BZLZJP}wDWs0#qW&OZ&6?%^1eG}!8L$fi3ze?SV ze&w6Id>hZFlMU0WI*c#j$MsZWN#3u@@`vxQ^cVQU;kc#+{2RW{z=L3@j4v#kkb$HR z64FLGGJ*H3F>rsS`G??2V`pt^j`;m^)D?cW2M3?sj=JK725>-(`eEtXn!dyFhq%fb z6EdbTX*`NiNG22GaBbDaZ%Ctz_SLrZTA6-!rES5EF4=sMvqL_AI`b9Zb2~ckGqpGK zmQ#0d-skhVZ?wLx|G5g!wbkdXH=#!oyp<{|+|J7=Q<)odZz^-J%@6o<;ycsH=)KC^ zkb!&2oW&X7SDgoeO%k7TtMikVFFP_iCiFW48O(rB`XSTbqiiK!Oc}-fL~H3&(OPpP zKJJ_XHo%OL&I+qDhEV3jqg6Npm`70;eQ&a2JV@?%7N2{aPzLTqTu0*!=4#MGU)9-i%RygwiL)gAK41vc11?rB^&ID_xZ`~SQ)@_y37!3B?z zA3-_cRDa@yhgnOOFKYF$krd}-ooRvpt~lpWjQ0(`ZCB>Nad8>NqyE&r_1Z=5?mV@5 zjOA^{_8$J{mWn^ITXfISe4|XoeviJz9w2P@n`OsiH~r;%PdflTq0M`4!E|3BjK@=@3OxYSH~uLSoV|QS+)%J!$0dgzXhM*Udpp#X0|S% zy$ZZnyT`pKhItKkmg+06E1HXT=Xqa}c9cJNKJ-l(jN*RccgxXg7R={RCT`ES z9m8_Y;9BR8w^OF*+J77Ew_Cfc=QT^mH*M~4c=T-%ue5zL+S~JT>dD7hn{#W91{Zf` z!%O=9Qo#f-7)%}(-phvD%e7-;#n(~(hG1L;9$L_Gsy|uv<-5?XzUgm(a{6xd&`~Nk zUggj!cIOMRukCV3mpIbArF<2bd z=>Xrh>}j2sE%=6ut$H0MoAd3IPs}}|gv;au96jiAyEP8Q$Bf6J5qB`fId{%Kls$rc z&{y>ww{T4{Z3lXLBd#;D9+!{~qPKS!S5i|H~wwKnod zw$cgMFxz4)p+Aze5AlRP_4F(RdppktM^jU0Sa{s)oCwj8`PXvew3eY~@uOf*(BAP` z6#c1sH&I)C8)-}$KdG|fzrml4jAz*?vWu%WLJ-@}z`4mEbxi)?cj+AI2+3U8%O7$i z*%tq)-B|vH=UJQcX|FI1Tg}a}YEycsOn<`Z9)H?;Z43Gbd9}3^a(S$6ee>%;>dW+C zF4pO$3ur^~qkR(GR*tkb9G#ckCCIt#|)o9HgtIxE0Yl;U&ouxQMjv)yO4oHz;|ZB3VPI z44pLZg}y~-dkX%TC*pFBujUjK{|J48IyBJvP~H~S8e;4}267VW3FKSr#=_0P@T=eu zPih`$aq-F6z7b5=Y`i~J=iy401gCf%oSw#TPrgbE^9Z!J{a*b`;yi`jco*9F_!!`zim&J1 zD%}*{g?Dd0dPsNS6{(Z8M@IAedhr|mi`RT!$9i~&bru(ss{1C=VJe> zb!dDe_qPGt@wr_#?}EK?!h{FDI3B*y^Nb1Oukd$4-r(v{VB=YQr2bkOBV-*p#vhd&<(Oyi<7)1L{BHP;c5;iO(i+#6$she&d+_M% ztZ#5nKi{KFh~sn)kYrHsD~>!8PpWKedj-0}@knBxF9tp^yr8{V|7~{=La&x4=6CSi z0!_6)te|%jV%|yki9J=DOmy`6SafVMTqFPdmYr_%5Hw805Q6&N-W89D{*sHVI|zTu zJfyak!(YSNwcs(!UV8?anVOo%3^r(`_21B5(_W{#_F7wg+w6v)jBIz6(q&S5LUV|h zS;6<J*@@a4cNwxAy&b zJ5I40{jSDnV!bB%R_53cUSJhmYF|1`YbCOEEB!cIOmeP!&nROuto~Kb>p0~rYF?9E z&1H<5AH)x#Jl_45T3bzWrn~#%bM9(oP3uD~6NWkKrVbyaW|cVi}JJCO_YVU0P&H~957hIEm1`ToMUvy0h# zQBpsWyMOfW!2x?~EO&3Ldl#`L9pW~v>q2?{5Otz-en^k>)>!f!a+Q0dj1P$Ee30_G zw>ihU+*8co1{-Usipf09vya29t>iao?ed(#-EOaz)jC^0@d(wI&Q}~sc_W2_v+ea8 z-u3Y%eY5U(2nsoYhW{osndQX&q3dR$#=o<+phtG;vhcHfpVfP z@4CqYjDJs2etVpspt?VzuI=rl&OE=jlkpw_9&CZc&@Gt^J;NMAE^AUacb=15)=wp6hy-?F3BpW@In z_^-H>t*OF0_vRP=G!egHPOazz*M{nAk74C~Pj!w6b-c|N$b{Kk22bD%1@H(~%^?Z< zetX&+p^nX0>I<062X(%|Xy&VJ9*gjr@K+>9ah{cMB-z$p*veUJZBa?K{>h>p{BVNK z9R*hLk?sv&2_MN;I)m@xnLf&!oJ`aIXy`oNhkIXRJssZ@pt5KBdg!g*hshJj*XD5x z9<%?@DNp~#$tdM>#96PSzVS5ATASa)|8>$yJy9tSd}rdj_ce2lpx{^86#O#+ zIQpqmrfflDXpV-?rmo2Y^$Lx&jeb+R*Qrh8pK5-sV$Zg2%~=^LbET^*+#5}%&^^TRSq1WO*?B}4zSg(-Wqhqd?Oh5S!)Kbxg)Q z8kPlS@!qE?7cGlQ|Cj&CIk%Rl<@0&IzGIWqv9uw4{8?jU{AN=-`^R)W-y@9$Jm_1) za^3UG-YXwN<(=kpaB!E`pTeWH!+W(2U)UK#S}SaU=8`4VkvzPDZ_WH^EE|A9^>Xx~ z{p)|^xjpO$zM_7mAJg5wFTQ2Pu^)*fa@Pq~E+;3jXh60|}OA2e7w*ld&h@ zRhb&OYzuO^OjpFYT=5ww2f%kadvD?3IpADp6BNWZ>=yuMaUR$6`_0zc-86Kls4~k%weXDsXJQGQIi~6Rz z;!E*yfjFi84`Awi{_NS8=b~!P7UeZp`kXE0)eE0$=U9HL&2-MzPae{IFJ7$OBNWDq zL3<_Q2VZ7OH_;#Oy#M$)=wk61@sIcWpx2eWH`(NO%#XmD=Uy{=&pULJjhXKs z<(+agjn`=BG2xN-U*v8)Ya5$G`&@c^j7e`iT4Basc~ptuawDx2cZSG|UbJw~FJ{*cIS8v}wdQ@Io?wlfAi>xI+eA;x+?% zjHXPnYG2f&IdE6<4rPxe@Y_>M^L2G{+x{0tBo+0`viDj zVekjBA2@b5U(r~4dd0C(uVNuK zf1rCpy|wCuRP2npiVJJMw70b^@0zx%c6GofLH}B(@pVJpsr{3ZEplSIvF*DhONvEm z&I(~k$Jg|o{RAWOlE$dmbgWkb-B(YpQ~W5{G}mh06Kq%Lj1R+8&{j`q-^aAl7`5In zKS}$2Y`tW0G)H*W@74Ndv9$4tm9uv}=XA}V z?Dh#!ewE7ON9dc*Wx7q{b>}jOP94^s`;Jd=gO%SmZXe&E(1^W(Um}~3v6>BPb}ao~ zn2!#(c4_BIwL3GmHOe!o15IsQjH89IsV~#v(_NYAahZC!Fq_%cJy5Vy|6}w4E-HP# zUwt;a_L}4Nd^rynJ}Y-HyE5cmWx=&bWt`op_@v4Rt|<=gFY*0Nz;J?tV}Z(Jcd8GI z2Q|gEmz_nC5Aeiq(I>>yF{+E4T3xeYYw@Ig0Mcnz1{=b|lf^Tu=tK6aaP0AP2=&2} z#Xg0P7PW_5T6@T&+SB{D^Iq@t+hQ{43M&I&XW>bEUCn;exZ!gTm&elqe9!YuZM@lH z?bUm!Z#W)(5z19)PlfNnWo5n$-rHUjzRAz8iJkT&?FiXU;#>2_7;k|(!d**aWOkLl z;e$9{);*!}l@|`4;_xB(tquC~wz%fGUn)ny-vv#(YE$bGEy%z5dc2dqRoMrWKL8#J z#FBh_))r;;UB1lWuD>vrb&BwmH6F%L84vK^?PlRkK3Ryj zioF9YmG~vFshsg#ZghG1qTR4D_7oo~25ohqX(e6;d?Ejayer(W&#hul$PO`EU`Nym z_)B=N!IS&0Zx!6RcfyfqQF*_E$7yS?j8Ws&n$rNk9bHUU*l%RGrP0v~*_3SPy~PUk z-+hy9Q4hbe$tCa73*yaAxA$1~+wI1itwU?v|3WMAKWJg=ebDcI-Ye#2>j?DM&9nRv z^{>9wuXuR4zCAyP7i#3c4d2kwe>lHWGJn1DRe&X7`=@APu@~C&JgXc~(KZ1eeWrbS zwg3AGnzy#C{i@9x#0=``3gD7XDAQLvHnU`^N$`Oe!!d9gygY~Uo@KWRUiteY!7JQn zt*t1&5Z4Xhu7j7VEC%*!F~g@1ZCGpvAK$YvV>kIb7pF`Rea;pl8Xu~6I3iLG_A>Jrae$bap{Rcu-7Z{kra z1K!@v^8of6bg(?u7T~#rcNVK_M!s}j341P8{$k!Ue&@ed?UcS|;;|1@_w>rPiii!4 zs4JkkQ`mo@lGpBU6nAft-Z80GaSP zQB5yS=J&3izhmPneX~FF@Urz?&rSVI&vSjVM;BK)=|4t}s}wWJWZD_t#=x!(_4;sS zBkl8Bw(Q7C`4D%`=2A|Uw=JIdm15ehL&xdd#Qq7DGnailPQJGMd&z!G|7u*p%f8vl z?tG#$?O9pm>FN7NzfS#cS^S#88`9Z2%cF)D>y-6pNA{s-l5<`%R$y~GdQ0OylJ>C) zSGP|`mx8Nb$NS)7U5d`!$g|g(@=e>s7ZOT&eBL zhk?V}Pc~0b&*o0fz?;@OsWkWPf&b>(`Vnx-CX^4phW|z*?8C&|QG{lUYbxzTv@_hi zme%3;7pwyV_Z|)LdG-8Gw4K7CpXbz{XtXUh+bsI3k*5*be`YOtXy8`(nmUiXEdDn0 zPWYDZJ#BeZlHb$YU|)Q^C0{JgW7v z)(Y3NKZ4C{;I!NXux+air1)+P?%Kc| zJm+Dk#0cEnT{9nN!0yNUCztD!7;A>E4(IRx_ltP`PV?yY&fkNyXL14Vx2_%i{9Wbk zfDbapnx<2z=l)akY?@vfZFqc%KQisi44nBo!ppbj#kxm6mDgQ&^G^J#`7hL6Gng-> zr>v~gT_5{l&gNuaeg^uh3H_xr-&IcftB8+pcyx7Zby(Go691ULq;q$i-qJg}D~oqJ zYyP~T&j?-nD`+FR_yzyH9=lrff*(U2=5>$qW~(#`WIqGHigzl zcg)V&GdLG|ChXA^zC^b}(`D+6)p#3vc^p;g9LQpwG5fmfe!gS%$OhW;dgQ4e?0h}) zU$PV7E7?omugr15_%rZ(68jT}qk(ujyi8acX>4QN2ON2MfV<5T3O8XE@MtqUDet3x~?|`8W>swpMg9F({RhPL}?YO?guAe*8r4*E^K^KlkAO z;r!p5|F7f!oYwA5OQY`b(|9gM>`S3d{4&{@6Pcy#rqQz+9B+nJ9&Q5|1D z-wW89VfXJ{_0S`!0S9ZtV|wMQXgJzvZ9d2kKlxkB8@w0mCu&=6m?9 z`tRcZj8y#^{L$Z+T;Hmr{CM_17%n65kpl+};3E$X#xVXy#yplWj$?cp3wm7V@O9t4 zF@|3-*m`fkCtBkpmbO6O-Uz<7EIoy@_fB55HSf;ep=S0X%2iiQ+JP3(VQ3^o>WJk|aZUk@CXj4q0A zB+s$0+)c*(E;`u6r>iQ23KU&xJxRNb8o^uU! zZw7Ld+$%BCc(8ALp3r*QiRY549hwyv^0MsZC6vRe?Y#|cKA*&*AyuVlIS4f#p=XZ4LXzec>){14`(7qYKp#YH3RD;b72 zvv^B;J_TO_?-^}aU3lYS#vvJz|26}ilGx9Wyd2Md{$2Ec%P!Z?$W{0A@2-3c>WQZ! z>g}pr4fsrcd=tNA`ze;7GTN&lp7^C;<5_tImUq<{*;=pU)cimAXftIL2aEm2I6k%^ zd`COE#hfqW+EO{`LTfL_eyRu5cdUyyQYL|iI{Iau+A>)R^h*k7ma{_N;_LA2hVG)9 z3m9wv67)3oS3B?ge6R7#cMIWY!)NwqH?+I68y=-jmJeEEw>sYMOP{G&4Dv0J{T+FZ zfO2VNFNnUHCuC<}8yRhtcc(c?-&+{xCgvC8myhp)o~=Dk;CTOjLTpR?CC?P-^f^Mo znVR<*t*{42@^_s+HD5P%KF-gf=hPqvM>d=Hqjv#!VP3NIq@IN{$zaGAY5IE|C|SDf zdn5b3OVediJe)pPV2a{Kz^b`nKYp9sNM@aU+PMOnGu^oYcUk+vT!B3KIaqb1%gtVe z=k?p~3lYEgys}EnIMCr0ogKmpzHQDzp32Cx_5n%uD*gEVCF*yS=L#G~%sC71H{vU$ zaW5UZP3H=Lr)a_tzgQVdPttyt+=$+?t74fBeC*V?{W`7I2Tiu9qj7s5;NE)Y3Sh%W zP0`RQ{CH1G{35d%)8`7Dmoes$-|Fcdm8mzzw2x4=bLxy0nu;fVevo7fTK_#i1-nBr z_}2fy`6+Mc!N+hi-tym+pK|)qN`QkASx_=%cbeujSx0 zt$OO`wY1W{=2M^7@{sH%XlXfu_=%6ncH&vNfMH&X<-4TvS`HDc#DM1VKg?@Ue5P+^ zJl}_TE$TbWYdMZ*vtd?Zdoi8`Jlk9ky_DB-hRQ>ea36a*uSK~q)&{ZdNAJsyMP}u- z>_t8Ct$Z8#{C|LMrYFTOjQ?EA6RGl1+NM%n)mu4CK1n5Cn6apAzxwdFXS|nyYu`?Z zcEmS|wJBdcZ|w#=l;Qzj?_Y-LUt<#56lpyWd+uzk=@(;M<0L zmOCQaspv@YIDIG%BD;BNH1$zGXWH+cS{pJgKge60?=o@c=erQI-=?`jym8pQ)qIty z^J0H~7VkuhbiT`%&?`Q-MQagShj`hE)%scaF5guSDs7owC2n6J-z726!vmtJr{}PF zF5DHiop~+^9V@<&-7VgHP`ZNuBh4M+gK9i(WIeXC%I@}u40y$-@Tc%x;oVX2=Kr=l zms_*sL;Qyi>*MP$B+q5jcI3IdMEVl`8_o{cmORycXwPg0Wc!$VN576x<<~$zkAjCn z{r2;UEDhg=w^gNoGVRsS-*!HzucH_{H-%V^oy8!X&ido*SZAg4T;9w4na*?h4tNoc zq~k2E`s&&|mroIQH9S(rVwe@(Rh>UAzO4MKpc8!f z_xu)g7(5|+;f3b6{OZV^lizX~bQ@`|Seh;~3OSN4&XmCnSqSsw#u7`~k@`b-NO=A6 z8u8@|fqfKp2c78sYbO%tNaZvrpF#O^ zoAOZ!zoA<>4$5y>_V~!-lRc8#iMelocJoNuRtt5M#w8nE`GFsm{$(ChPT#xX8~csE z8~Ce6~w*2m~*jCaTAzr_JL~Yz&@oClVXM7{cpOvMYohvmB-pbAW>2UCQ8%}Wg_+S%o zg)n*lZ8Wy>cF#q-CHHR~@w%Ne7d-**CGe4<!T z7yVkb8MYG-seVM;x-=RAXNt2_&c!W8C7r!vbmO=7BinD<5AoEAEuUp$cwHkowj8Ha zy>uPFMrAk#&uiU2cEuGUp6v{7_uF#u_^o{sGc-p*WBbjs^21eL`!l4Y{NDCv2Y(1V zv@*XtLE|u2rl-nJ!i^u(blPIxwEHCDHN-eS(DWj3oxu03ek#0?YBx*oEvVCme4!R( zY6fFezG4&W>WOtxHzq3wkJc@hVV72DKt08AL~r>{T4z>Y_Xg{tibdO6G~bBFJS$&H zGy<1~7w_{v!x=C>-=w86f_JL^*WTitfCd%(v?YBhPan}Uo(}?9arG1PK~|eRNm_cG zZ|@ut-c$7r#u26R6KoEHR`RczCoLC2d*5Q6SH(fvZw$wJ_B|ud4{u4A=~13a_POzP zeiN~}_x;4JUlUK)n$YhaqRoL$8;^grB}HM!usbeZ&n;8%Ig8L_^ICT~VAlbQL|D-#%1M0J?+{`=zt8Ehs*muFv$DXm@MB?a7~N#9QN^K z^{sCfrw1OIojou6;55FeALUY8 z-nHOiP3&CWhq+y5@3_70DtoH02fT;=o8kb0UT|<24WZ?w)-QZQ*&O-SAE2)B4bQ^+ z#e5g9iig8|>(5b6Yi&2GzRCu0XXaT8cJi#RbFfupLUdgjl^#`l;B(8;s$BOm)-ccd z0asV}+VZW@$2T-L#cc=jpTQ%3{m1knV~*FhPLe=q21J!ZR>639hH0A zi5^eb8yS52rF5ATzov6<|6pr**eT4Tzvs8jXOd4Ri|ISPeq!w>?JuvpRmW^OaBOoH z{QcLKdwVfsl>CVwLfO0bHQ92uO}V#eAF~c0YR-)1H<&L&I$`_x95inS+k6IoN6fv& zu8Nw_cL6Ow=*FDRy=~K&0~n>7WXE`WTXBsTPXV08SkFiYRs7JPjU8t9X0-X&YL_a{ zS$5WR@T~Y&Xy45Thgo)M7j=fSOYh*j)q__)-8H;j`Xly1n-5A^*`+@Y<~q^ydwkEb zOP?0p*rh|NPu&o1?X~dzSN8u+i|x|q9Bd}zjpz#IdfBC~#J-K^%ldZd%P3>*g0ItT zT(!D7ZClpJN?X!itZ(a8eG1{H-SKXm>s>jgc}O<6;Sbu$XV5(+2M6!uTYsq6;n;K| z`VCo>Y@SHDWqdQe2@Wsiw`?7?pJr} z&}^F)U@0_4&&tOBDrI$NhWgd{#%UeEEb><@`?*Z9)VICBk-{@RuQr~aWAF%j?- z+Gn-?eX^KC97@eKE)Qv z`&*#)Rp;Z>kq@KT5;8&zGL8f3`>yU;cdx1J`zdSZZ)=Ym&w@eltFFr1JI>!OaDHRw zrTfpeKEZ|hoH{P<)9@zPsV~@ttMh0Vm|pAK2;qh{)~;kqFuYYTAQxH>u=3WvYY+ZW zpR$Q{X63(8*3-%QYjki6#{n+oW2pS?l&`eOI8;Yt(OKW(W%&UbONYas`jT9$j@7&L zoZ026ejE!2FP|E(#R#WsU4P;E;oLi9^;EZ=`x>Xx|DxUE*j6XcI?KGpo%z0s=N5PF zxXSKE*(r-S=bhiX@_S$Y3`U-PT&D`>ITbjqUGO7+xJozr5+7D@5r)l1-w zGCZqH=NLccbK*1Hfnj4=cMzc;8N27Zdp+KGXKhFESVNrLEnM*ZSJd}Bq31_<7N06s zuK1AJ_5A1K=hCMd6S^3liQ!9($-%qU+1M)6#JD|P^=xOW^Z$>6pZ|B;|Cqn^{B^;x zNW5{*;K9&ilk%AykLddu1!$u(+J`Fs(YFen%_U= zuR>#uCBcg+8r%1sOXK$dkEgNd39e%qhBlt&_c@w}G(M8Ci>BvOKcw-QJR6P2Wzd*+ zk`v8c2^!bq+mOcKB8$dv7W~lVNc&%<@od2n(s*2w#;@W#a8zk59&$9E!MAtur*kp{ zFZ+l0)PMbLSz0~QAmNYaBU{syJb8XCGcQRmO=b>M?1OtatPRaqFQI*8>2S5F=UsVL ztYnD3R8MjtAJx8tuf*Ku>=%6#E+zMCo!t9p)m2*w`#hXWhk>`uJe-Di6Lv!gZ=dW4 z2e07&rJjd@ci4DzXuF_`!##M0PxOVxs(HZMDEHP|TVv0m{Lp{#_o#!PP0z-2!f`1Ppj}-fDf|s&%T0&oC$zzJ&cI3MBuE;=?irtC-0mU+*3{Ubi}3CZ!W4SBfudttmY3;q`|yk2xRA$CqEqyW!%?(uhvNCoKeLd;E|x>9vIKhVQ_)))LmZwkm!|o_ld}(a@gOcf6MH zJ<5b}#Dr~t{uEyee&*2HwS=~$kE^oqTBdybJ+8lfasAE6w)i2popJp}WM*ngH>f@7 zh8-2x|HfS->xo;^WisT=%dFsAtymoVCm;P*ep`GBn7v%2eoBxyI`un=L zqI6CukN3|)zh%hT|J1ntVF4`o=ZfavAw1AK<(6kmlAf-(*+#LUDbqOXyn&H}REg%(v8gV)k~fpNs47=J-i< z%>R7Kz_$%-1<80lS@6C`#QvaY*Lgb6;HqhyQH<`m>L~ty z1ii6k>3yB|=CSpHIh*@8uF`qMvS;Sq72|Hv)~$C$&+JvH`%bG{T?fkUD_1Fh?r?og zoo;;{8TR#-eJXWNs?*n6==`u~%^!KI@KfzrGzRYzJmAiaNch9SxWlrWvdZSiadz}R zoE^O{XGf1M4HWaG0oI?ik3)B}4zPwb9^4&&&%B}IbJ&EOk=N8DUy8UIeoVx>8|9ZY z$8k2*+0@KkLR%I+vgv`Sd*IOgoT#yN=#TI^=7aRI&d0(7;TkLq24L2LvK|ZC&je|n4>=4bbXn2O)e?3k}`_*>6_LS`kM2l=jPM) z^r+k91HW<8be*X@e)Wv~TR&^{i^pcrllRT} zUgF2`+>8$BpGAJ$Ozu&q{7rm&@Q!#dx617ftZL`PgB1hM^&ZcETknF_{_JqAFW{#y z5Zw0x@0O)!aVGScoC!UyB>C!_8^!seABwt(Q%??`HMS;TR@^##&sh?QXTkCm=j*Ev z!P5d9_8XX9=6?6hZ8iT_F?G%RhR1yC(CLh4Yq7dMuY0ZKt1Kt)ZQ6yiNnH`*yYvSy&WRe9J-oMg8M^4af%1~5v`l%2C%&rmwBt9;ABr!Pg6Bf2 zot*Ue{CVtGi>^BAE95=U=3ewCdGcpm8}9_q%Duk9wOO9UI;5ku`pZ*ad}8bI=Fe5_ z5c#%0yghf#RGm?i7ysnZeNjny6f58%`K;^dM=>JFt?&l$Q#-%DNjRmF3=R`Z>+qxVI=FTS@SmT%GVR&Xgj zZt~Oh%F=0RnmeZ6xRpLej3 zvE0yg-Ox$O)@h-nW zGTlkrtMXs&l5SpAyuK^P8QC9qec7Ft$ zhY80WwDqspxLU_)ik5XV4-EM;xXYp?ZOm62@Q>P14vXl!j&j!tw|sw}>#q~o3ysml zPR8;e^G$nzL+Q-$S@q7NUcupHp>G4+S{vZ{7PSGsPUjo8)#NQpPe4!SO;2l$a{}*q zuFFrGXD0dSK4MnJUnzcKPtoKxJ&v|LRobo&Y1?x{*ECPt^~8X%Cw@I0d^3;l;b0e? zIw-fNr^}M~zJ^Ct4!Z2(%TfRNuchcR6I@gNDZO{Nwp=RcrT3>&H=Ku!7n*+f#WK9T zmHSAIzH?iq1(?=c084FO{L^0N-Wj=$-g$9lxhIc`4Jj z=%Tw%HyB;2ckvPbhp(&8f_F3HAWxEgC)f-8*8LJ2_%Ap+!LiX8oZqCj`1Uy8YG|B4 zd+-|GC+5R8bVl34!S*F_{?^BZTln!V*S6k&iQnEf(eqk+mL3Z9B{Jdng{aILSHBM# z`4GQD9jZB4d8g8yM{?F-NC(3$bYJ?0ScaHGD!EAEe9-y)l-cZQG`+Q+cAfy{9QI#B zwCuCO74XJ1)Eay)x}?14jr(XmQa!`j^9yH3Jclv7$FLOA9pZCguW+<#w>#WyS>6X2qf1DSy@A8YbH>jG@+#Qf~q_*+_!-A%mX#w9zZf}b|= zQ*PCkBKPu$hxjIY<*#);wv%z711ulO<_2IE|0`y~Ji6WMu_vQ{682mQFT%+i(`9<9 z`vEI99Jz5Tne&wgCEMBNTJ-hYS1B)+7+_;G)Y3>^QB(ZC(7_!(6R=0npUTHnJWn~1 znk$tj{3^y|_84`QKPsPD`u4-LCp{-!%bKP6a3D z9sf|Y=%5nz;8oMUyHDx7iOWuI3)58juZ$K0FL13lTgb``nIk`?uN3%`v% zq6Ikq6!V$R5=iKiHjTsIBgEZ7@W?vNy|P7}oum1;1D=~pdzv$Ko|@-#-TN*->n3Ok zkJ~q+k@68;UT{SgXy-iIknU1Uq=hq;R95Zly~-6hLsI>&m)%jMU%suQeey4_G5Za9 z3hyHd@#g0ouS3P-UXS9FxpnON@jua8F~CAj@PPy4VSb0_4ENOH{jW8i?Drjav&K=D z{O#a%pgq2V9Tw;r?=RY%56>>Neyj0h<$?%?SRSIG)A*hhPyU=}g->>c>PvP4xQAN@ zx{i9rd!iw-5ASc+zT?Ml)3|EsoW)mvE9*e@_oBo$i{_Y}vG4dl6zfOcCHpUA-*Nj% zBdi1Et1%bPr#|n%+o<9FP}{oi_*0BSyt=KqLJzxo(p3>W`=Ib4{S7`I;CEK8&|hNT z@%yarSXTcM`;Na!T-nc6nol$be3^HlFBhSg!#Q|4zNKF;S%$r!yfDS5(>|AYH=aMM z>nz^h{Cj@V_T?w_7;W&8w=+NKSI};1%2xJv(vHec`e*d`NOto-rppYYCj{RH);J}f zvPITwJyi1<{3ZP**-z&uebd?x?8Y|qDZH&bP_uX7Y1z!)?)YolcYGRSG#kzAuIe~e z9F#3*|5Njm-WSmFJU8Zae$oXRb6^*QvGk5{QLovU@Iu1A&fuqnkAU89KQ^9Yn`HNx zy+iz#c9gfP=TPTm={C(Bwr2wyFTw8tti##Hd#E3)o3xEX-&;JceyVnKtu9rWs_!27 z75G)UPi+Ohh51GJ7yit?DHGy2w|q9gkFW#(RDOlzOz&kQn(v9sv|ka+&s4#Tk7D*c zvM0Mx_Cq<}w7P{ht2}$ms;zA~i%EN(-+~W=|MKDrtQmOnHi6ooroMn7LDv!Ln6EdA zI$1X5f$A?!k74@DwzXVN&Y2!-P{cFaU!Mn1V3^(vXk^6zZ@<^@q*%)Rk}R*V>MyJ<&}(|L4xp+u;9}g|D_`>_}r1|9-PlHsIen zWs!f&LwR~tY98sSf7i)s+Si=WK0W5si_`}EISt>tAHLTa_gQzH+^zS-Rn`%&XkSTP z)}}0v8$akj_*QNP{^{6=;PgSB`(`>G-;;GLaM(eAXQ{90$iz(A0f)q14ENR^V}DeA zXubmP{m9m1YR7o=_*Gkb8sc@EV4Xnq9`bDv>;HmilED+E&uGubWCY{Q)RX@#*z|n9 z`Ua+T>K}Z3k2T-n^qT6bUc}nxa5~NRt+FSm5087sJ05)ac7CfhW*c|Bzx`tCzrMbs z9rOSDsiU^nS^0oxdaC}R#wdJh4mY0@*s1@_wAMXJZ)UB@=L1Mq)RxZnl0H^$!5aQo zbyWaQ(aEpYwP~lKtJ-3|s_Y{Hm&^&Ve&bt~&c1~E6=V4e;P>qVYo+a;>X`g|Az8O^ zF}~9sO$)*OkMrF6j_klQ_&$=3%+hzCpk54LF#Zf!vg(pkQCruI&1jzs#y83_XZ7>$ z5&F@dUi6@?>u4+$oVI1tPN7_a|3mp>UiIg#6rTFnBeot4Ka71@ecM_# zg=ov1VR}~QKk2#3gH;{P7;FxXqCbDO-gqP@(jk(Quc03jux5;BMMfEK4-4;=7@Uv4 zg|Ro<8?J1WyZ80cvSmkB*(h%o4>(dLUuC7 zS{YXcUNyIAy|UU?z*jxBa$Sj8!nfZ4XvxJouk82c0+aNW>@T&ev87=Q^^Mn=5p9Ye z8X7bF?hIJGy!lw|Xky*QE9hInwyk$jgtGFwm+qW3nQvBunVr}Kxr}!Hw&-bz>KeqZ84Q=9~bqm+BLf7-r)QK(dS$6t zb=sfa`)%@)<2C24`CWT|Lr(HuUVWr=ZtY$xr?&^!_To?N{3N z8oaG-&%xFCy;xt}6My$5_`5F|T-mcn*G>6725;_}+;vNS^5CsKle$*5P3r0_P9lfy z)WLsJPS|vOs{Cn#w;m`N#Xg>{^{2w-XxDh{RQ@V9SDgR&_*lz-d<@v-BdCo_X`|dW zuJl#jX}sn21CD)OB5zB50jw~uejwYJ*#>e31Bz52dfeXk!|`jTs- zb7j7_o$~#>+ufDZnoI}p--In}_~%*cfgxqA za^20q51%z`$+51!{3KiKaPJtuHO&DK3Lz9vE+V_E&YCNnEO4p zH2sG@_d7@K_ww%Y&%l>*xoFq$N4=L)#&W;;wyNhEk#CG?KKl^zxW|<4xln%-}vz3-;zx1;IZ2TcEt+#3#ox7+^){(tqq^NpoD z{crGbhx=WgdvulFjOQh8{H!@n?(50hzJSLrfVTMJ9r;rSr@qU}&S`_?4}td&wGMp| zeDqx!|CT@B_g+WOwtT67Nxt+A-c9xAr1Q@9c3;f=eR1oM`q;&_Y4G#Ci}KH+heDZn z5Ac+qZz!<`;pBleQ8zixI#1@|@;m7F>4s7tv4q3OH(9S3*b@BecMvyd%k^G!WDM`+ z_>DeCynL4Q#LtfGo3iZneXpbZT+TTIAEqmJi($9*H|(iZ`Mt?u;4gm$IU)_Ea#1pH zMpyZYsJjDMxGDd(!A|(}$(8XwgIn?o2Jw4Joz3}Pvmc8Mr4?5<^sYQK-+Sw%eD9|i zTRhHqe}v*A$}Q~&4|n-*&A@+_@||)RSiPw@Son>>AUr6~tM58k)vKkeU1J&pAU zJ=4yf{8t_by7P{#eFV2NR&;qu_Sm1^`yJb-6XF`5D4}oC`bF^mdH)!%f@7?MqvYTh zEP}zt2|UmHvQM`Toya(*D(AbW)&>t_j6J$XnmH+)i^4iK2P(e6d>=9AoHN$Na;WoW ze#?hbd!wq~*gEv4p#B@`)o-hBKXn3cls==r7~5*Y`Q6&9S98yCQSr3GsxA43lH$Sx z@OGzoQDeeq^RZ*)F5Q&hfAHqE{RTHQM?)K#|CD#sw}$*M_;~S)z#;Rqa#s|CRPN16 zV#zm=i+pqboWWb#&K}$duWd%p4{ax2N8u_VcNDeo?v9yMm{DN#oX@$Ie5eUY)t#UYvqW!Ef+_ zpWCIJn%LILmtOu4$O+>Ob$92b2h2VX{_O`B$|G*?fp*iQp||v6mrcj-lYJSrSlrIX z8kMWOY9)6?h(6H5m-!&=nB5}0H|MxBVqd#QqHmVVnVr?zy>@Jz7pIuwhbb?5NnUIX z3?H*Izb~{tYmjrw;N^Yn_hl+~Xk4y+Nv^lQEmyiq@5n#0y4Yj%Yx#Ei4nu-pr;8b{ z_)vS7`e(Hcalm7&hxDH39C=2cb??-c>VLsKKZ!23@lduCUX$(FS=_H{268RDK6B~T zYbR1i_0Y)``$stKy!71Jo!-u*{(epusNOp2ZCQFEJmz!c&mKH_CAz6CDQBJs4jIpQ z(mc>p)g#{4wKXHgy8bAYXS~#DZ;pERhlgr(LaHz3?@IjU7|JJn)>L2WXTI@0ebAr! z7&D4KG@dp4N7uX?AK^oz=;K^$b(1sK$2)kg8Ak?e|B|YAMpnIH#(y>C6XQ>n)i|D{ z%;SaO%Y`_7h%t}Urf7DBU{BFH;fwS%^_H=? zA9R?@7jCv6&)}W-V?vdcIsP#j(DV@cg6aK-UZ~{$$s#%to_1$?{KpD4)o;u z#?k=xLwODJj?)PrrHskPS*`5FZ!DF+84bNt^{`V+F4vIzw|gv?eboP$zMbaUJdM7P z_gTORo;ce!-#h4Kld_kn>?yA7DRJ4C`m&5`ruRK4`*xK*$(20`{YJaxJK^U$TVr@` zkAK_wJGaH0q&MjeN`Vt;jdvY<1tv&ZcgT?{DN?-({}P%gBFpIuTr6Rk&}STWyk%tt+o3pY`IQsdHJcGZ40_KYY(kprN2%7H*m{F zdqpn7Z++3~S2RXLFK>#5W=@EP4xWfzxd(Hn)+;6-TKWk3i2RGm?QOd|y|;Up?dJ-f z#aFk#f9tjAanpgq>G9x9wC_ZB>c7g|LK)$Sd0^;NU^kvY_m=18y3f{F(Cq_r=XIA) z8B-eAGp9YWv5k8*bUSHGsqgS;XesLp>ko&|u4JCN8lBUY>-`1yF{zLKLtBR~mz_H; z-dk9{jJ(myTFLnvQ(8ee@283$XQ{vGuD`3ZSB|CAR-eau(Ge^ISi6 z{<@{~x3Zykjqh)|>u<5^@4fW*Uiw>~FKNH)iS!}3`Yx7Sj4AauM?>rQwvPE(q(uhM^Tb_V60O`Ewl_7BwC+S8Pfw>L_*oX^Al?L59S|JZkzQ=Zd&=gXi&??I1f?vqWSb;kTW?85c& z-hn?-Uu89KHBryT&A0jdHhUj?%3@5|EUI%9?}^_{4$se&4Rv~*v7gSo7Gc{abbWs0 zasQziT{_ITm%xwdaeq*2Zo#;pr;U2!-ppL($BhA48TV}FrL1uu!uzao?~^g^aGpCL zJ+G(c&amuH(sN=yk(=4ofi1g8>z-2&E1e6U%T~1K^A9VXspnmwmB#PaDc}p%EbbOf z6E-e;v&bLeT;7FUtc`UG53iv98vq-sdl%u_wl*4}PhTksM zUN4nX`}C`K7KCyMGhQx3AOWLO(J8mQI5@XHi>!d{~+@tp^{vC6X&6hK= z!O&&$g*%Z8`F2SjKVY`jrgit8hvyFZ7O!e=ZeKID9zNoVV$}QCkI)m$IUd%h$aB-2 zXnf7wwrV}T9ePkck1w;8G6n~_&-->ytKM8E59~W_8RvX3>gTrbZBN(cp73Je+comd z`*ypl9P_`yxg_@O_SU=Uu3gT%nD5T`mT$JdzQKno-v>@4|MKnR%gNS}eD?GEHNro9 zYGXpil-J%;*=LgLok~0UX8TL<=M6?;Jbzs|z$0B)U4j_!hX=a8>~ zobrF_gV3%g&ZSn&_)E*cA+jL-qdl&lM8_UgXf8cL{A8E#NuqdidUSz1p$>=x$V z+2Cl-K}(`*4q6Os%+*>u4&ji#S>)#AN88bx%x4k)bvSpXb%K|*OBQ8}x2IO7&0b)9 zf_(ygsJ_8)5q2~4e8KKh2-ZXEw4vN$lXcpmPHdCXmeyNzCt*S#7wg2cg0}rV7~1oU zOq3?kW}=O@s&B8mB}{L?AJXf{X6dE8GoA*9kmmPecV*^Up?~|AIGQeuN?Q9qc&X^> zWW(C2uAOzModAZfi9T9mqHpobzh9ryQ^DB0J`HhiYtPn4fa_w6CqKuR8Jp-)nR_zY zOzY@B+#o#@+h-B<{4btke>mtT7>CAd@g4N=-`d&MCx~VlI1B0Uw|2I*_PrQi!duc` zIc#amgXLK=KQAp;j^{?hW2!G3;P^Uir1w($eUg>713N`B5$j0h#YOQxGdC~ES5f}H zt>M)>?|owQtmZN88$(`3eV6k0_kkzem?dWw*itg#Wl;J>`Q)Ka@%5!UEA_M5?5Wki zVZO~LiA&%kB=uYdj+AR9oD_cdkCoiCh46Ni+V^2LF)&FV%lFVTYcg>?IH+k4gR$ zV`WZFj5UQ9jd{zLQ)Pm2`dF%uYjQRQ^I>W|#o;P}zo3mBrY|#a>GuqTx=y&2u6!}D z8lH1DC&oB$kU=iF8?uX8yEx%T@9vj!=vtmO6tIBG< zE%mCkQeAqv47A$SS9*7kF;=Ro04l4grHdQ}1v!}7_Cv@SE?mdcWc<$g) z``6NNs_G(R+Rv|?a+OK!jV`MHF{~HmILqmPLFJ$8Orcqn)0tUnTlM?s?_j-i`l!Fi zH)wk_?Z@kHvNIQAXG$KbXS8f>Q+%U|^BGtt&gWNq-Do`0#@*rYiFjef%J?2}vypnB zBYl39{^XCBn{%b}`7OMOhwVH#?8zt}=e~HHXpH+BAIJQwbFsXwZM=^hbjo(6oa{mM z{Tr>RgID7x{E9yMT25cOV@khei+xaVARFF({)nEXE0!L|zL9t>P`v%~FNwFRdeQ42 z>5FyL?*nJ%`@t`wMZyLQ>UkZJwtpm>nR$I}*jC^x%=eDtL9vY`{w*N)g)mQcnEw1+ zSfjt|)RF9Jj92U--U@87RC%M5^o{mCfLG~q;Zb@=x;(~dc74g3`Vmg0%Z4lK@fX@c z($zI}iZL9bZB?ID>F3+=a7WXk(jIrm`ar+6Z$)iRDR6E+zx`aTIY)E1Y*qQ?DuaKS zq<34@=GT~1MmU+QHmNthx3Pqac3qLhEl!Uv#zht~KAKm5W4?XAL`{%V*1Mk>#W9yLZ+2PsiMvD`Gdwq|VN2P;B z1MsMJ#Mh$7zPz(`kz36L2V&pMaPPwT@q)dQzLclO<;2ixV!saGo}Tq+W;P5k=$zaH zy)$sQe>zszQ_Ig@uXs71Zgm2=%V;;&1p%*Qlo9`C(kxY0{-QrJ6FXUDdC&T?altA1m%tGlysBPw#^@H72%kJVN?7hXR?8Lu1EOXx;p^TTgE59+a%+=sBLoRE|6f^ACo^1PNdv#o)|09(+)A!*0ouS{} zI|3f1XO@gm#^d_1k#McE@RqfdI#{cgZ;4O!Y`jKM-&X6_`ZX|M+jiH`0DQ|I?#CDQ zv4t}0Y&y@a&=_5!zvFz|Qs=kr5na;ipJ`Kmrq1P2JVs+E+_z>>&)9D9^Oncg>e<`U z_{*95l2unWVi?O(++~^as?+vc29H$sE@JO1hN1R!=d|Lk2ReAwj%;B0lLo8Km7^?p zob127f^Xgz7@)k$wKT@84jKT{KQfoAU0)d8Tg0( z-ktgMWq6^i<$DMo;xUI)Cyc4c7q)l>>zU-~m1gnX>M@q?>$2lp<_lBr66-gPkL0*_ zhy7!*-%4HhV(22i%jZ%2K>0C>YvsYa@_?0_bc5=vYyh|X65oG60 zBSZ^msx~{+p3$*-)}mrumAD>duJi4JuQnT3wl2`vye|{;b6}6P!OuZ#Bsjy&&K4wI z?9TlN;$Xyxluz#S7_Jb{0bABxHDA&<@HG*x{B1;NKG_Jn+Tw{69Xo%jOxJnW^(NBV>M0#}~4-BIrCx_<)~eNAAw= zerT=umD*N(S+cCN0Hnu-GxaCAvfTgRKEn$sBbv!KpE@ln<*3J5-_>_K=N8B3C3qjd zK>yZWE*iQ!hzGtAS|#uRZ}@$D1}pFh{xb!a@n)byL)muxp))YoGk!bMf_gevM(gc! zC&YZvGBM_vPM3Eid6zz$AYI7sXd4CREYPWVu=rSC1^c@whW z&ngwZ^lmk6*gN_a9YbA_MQhpnc8>tT<<8CanEp1LxN7 z`UTk$$@La5U#e389@Dq9t>43*RTcUo;MC+zG6a7JkBX^P+RVAQj{4Z0KJtPW!t>laxPWP`d`KzaGPD+j2pU)7lX+mqppmF>8 z>Rvy;^1cb48DPv!O~kL5f1;*nXm&uigWZ_Z=TfcFm@{DfYn>+-DaH=3=NQju=gD_d zC*=L~Jb3}%#ShXGPk(;+dGgl1)o*p4{P3Tp!{TSnlQ;4`Yo5H0a+)U}QGNOu?L2us z^#q&hNPmgHHBbIb?IHWattma|>Iz@oHm;pGPlh^EFan$6CR&T{YmQ1Li|CI;d@bNR zp9>=zNbkr#QVb=Yqk?(6I67az<}KRM{BQ9=>$|oc=sIYmZL3YiI_KR!yx-(u)Lf~tM0NJgshq)?)A~X+pHp^!B{q}+ zgLq1F^|(IA+YT2ATA`b?XS|=a%yqNy0pMlR=j_jt51>79+S^rdzJOo9#aHMqOo%R7 z=GJrVd17=)d-#m6aIbq-K80-DmRyVZ2k_A3>bp*RNBlK-ccH!0zJqGHvb~eNo_=R? zjt*u1%k(cqGv#?j?~6(`J`gPFP!KpX`(9mqgwoyka){;mTW%djAjg4fpVk^lu*8 z=)b)KFQw0E%{$xfYI;qE-PKdoWopM{m3Abr(!c*8xZ&fZ9~RTkXw?+ZDqqdD_cY4d zJNhJJ+5rD=1W)31`KxA|(7)_8}KgR z`7T=j9k0E${Dm;8>_D@fwNB1@)A*V_iuAki*?4{m=hrkH4d2+}AEdp6PN_rhO9Z3Q zF0iwQp?3#4?ySpO9rdO0$Yz_&JHwCk8Edf#UI^Or>$%~aVQW0Jm8g?~H`L3cStIc6 z4>uo4|5RvQ2S(Wf(!Cn9jW;+`?KjwS;;S4ne*6jW9n5(t_|=!zIukUhV}mxmIJpM! zr-}8?2~$pvaLPE%rcxY0&#}IeA5yis8EdRlXMg75W$U}1oBEla=lW*%Jy*d&3g!^! z9q5b%?5VOE-{)$|RP#=TlXdMAzXe^q5uH2GjGQ6+ChN$34*mNp<^LepKP1jJWBDGH z^DDKcD1MyDm`%4)<~9754R!=J0r0QR+FO1rx_k3CkJExhYw)@BzUOE{Ef# zii}e>L6bnYu7)pHp))$ch5Au0q|XVvi8=6Q=D-c$c_XlI3Uw^=j?c3S_vCK?Pa7G- z=9G>N@phWQb1Zj@xxM)rJfb<{)Qn>kFfJ)zlzSfk5#(!d$c896WtVFyvy1SbU~_Z#TT|+KaJro)`^^X^x{=p z+QyVZ`X+50#mutpn_Kex4BpC_A*(q%WEJNib#ew$C$@~8QGiS|<&cFFuq&IeGq2Y< zB4@F`AXnP!>+!tPiLFy$k7{{l)ZK~AW540~j{KQ}eQmkY8^F7GopVU^4cij@>3ov) zQE8lOV^uh78*_l7x+8a0lzJ)&ruGbxG2Xu8X$7`;ajBchK z>0l*^I58_$5YHRD*QlB+mIhpfD-pbjkW6Vy~ zK3J3S)1q!W(lzIR3G?VlsLD)kF z{^(DiQ>W^ws;8b?J@r%-`-w;0`h?}}C&k-ZN5D4eV2ss{$4<_YFWV!2)_t`_?2gUQ zf$+XL&Aojwy!$wpo~vh@iU!QOwHKGinYH-%#z41C1^kYsU%^MB-zVS$*WV}&SNeM! z{k@I;-bR0Kv;8$Td+zY=UoPSfJ^KB4D>97!4r2cnXIJ`ZXd4>TJ(HqY=@4@tCA3G} zVVSlz;*-#PPj=&j(lPfP645aipnr^?f$y7hi@RH1&m3$~wg+3eCm09{Ur~AbW9-gW z{6p=`@s_0u=TJ_*H}2-`X}eTy(hu&$Kk%B3a`^oGZ)i1 zOK&QU^n2_<)g4fpZz%mcydSJOko!(A{ipV+G8TF__6|7h*c9|F;B33;H_R^#^WEe{ z_79*p6({&6d`a)_=Oi%jt~cj1)L*D&wWcwR5rXvVeHyroIn5 zyzqRl%I7l$-93E(@m>A17L+C*VCVg(WjSjgcy=P`s;`Z96%R;-V{icNJ(+fqe}Cma z8Jilr^TcL*M;GY>|OA(z~k(KiE6j-Jj1oGceFVp z;IT2ZZ%($fzuI@4ZQpU=^>_4@JntvHxQH=%o^R;a{?^fjEAZC{KT2nge>z?G`O~(KJj<@=haUzQt5))wzAODc(vfik z;N)0`6VeBNN1N9o`<~~29sl22fPaW{?0Wg1n(7JGfCqSe06e|Z;YsCwK>nMNQz_2- z0#8#d%mKa?*K*dwV)~!we<|l=^fvX4b8UhDyYk~XTdaw=asKmX+B@e`eOs|F!utXa z8*0gREZm$vyt}1k^}@}_>QjT@hV})c3Y+n_nCJ1M3!BkhL1$3tOrJT_-x=ayaFW(i zQhle8F8iUlqosguxZ#q|Ehs%XKUEr=?OOO`zNu$;ZBx%_z`W#(qYF=-oC^Isos$bs zo|Y;-ae4|H;vnpsgV2==rTcHlj1R}RE4`4kiRihBw0%PO?W7YK=Ly(+XJPxEg`K=; z=o=PS!oRTrjB`xaX9%_a6t(t~ilB>9m!1eu8kPG1d1@Q)B^o;T{#zT6CCoa)DwD1gfKwCSeEa!ZgdOqzPPQRico3?y* zn#Zqir9D3Xo)CL~Q|uj^Dw6+@*gJ98k?&?MBpVo;g1rJ|`SEqyFC5Ar-%Fp&Gjjdw z^v67dg97QyV`>B!-Wh7T`mfgYjt+rfk2D1AN8vM2Q}+oyB6&J^QJFxqva zty}rNX3MB<=^3@<5%i4aDH`Lh-Xpr1GqVrA@~(2K>v)q+3^{$c$+oF~IWi4CJ40>4 z?pkZ%EB!LHiSaXZzy1UZN&zjI7Y4pc_=!fP@vpu6OLl>k|g-dOFl~=kaJ%CTT zoizaRk0QVNw4Hp%q#fO+3XgB5e(Lxrb?EzOXvp`=`u)+o!GG@u){iOg+Igw%gW&vz zGy4pVvFDB^J?`VvJX7=N-|8&Fw0s4g?=D^OuR3}$b4|ucen`pGgJ`F0VU2sz87>p` z9lV(}yakmoWZsr!<`+Y7beXjSg?tjf6)@^vbqT=*<`#1;^NP7s=Rr>ki@-B@n6sdV zcXKA7bc^Z#3FuDbL2+qQ&w?iSg?_5O8|kaz2l}h`C8~cN^dObL$zfY{A{&wnq zO{HDJfn-h_c)3Tm?QOg>Ues;QhABRpnHLD&rc43aGUtSRd#U3~?CLx|j0vf}jjgG^ z=f93j%p2Zg*`}><>}Ysmt!)GK53YqKjtokfBdKHhf_cT}V2;{7JgF({^Q04dHut`} zoAJ2hR=aOb{Z@Y2efU(m2g$?tu>(15_6bcrlYbqy#o$cebay=PI2U|U9B0pBd&kq% zuP@M_C+xS))G>zoPJKJ`P}*Xi&kTB6NJCF7Y@42%r@FeHKX-1|0b%Mvkk@C1DxSO7Rl< z+XAh$@K1k?|D3ipSs20r{yl@^&Cp>>3f@d*OJjf|_?n~W{Sd+ZFWZ*Hp%aTUS6*rQ_0yI&Q08I!rLm?@2eqd%V+P@Y;tx_iCqW}!^tJdncz!%v`ku}~ z-4Q?iHNcvss^KYuW)F!wOVN4&WCq=$T>4RzOk7?hJv!&0fjvs{2U)1yW+?a6Y z_wrGNuanj>#nNDYRN;Q!%WYjmTQ`JlRl0lvrmc&!rHky@RXbw1|EBVds_=&!6V6X9 z!yUYdxu*KZI6g?aV0QE_?jE>lLC>o?n?i zo~3f9bN(IwPoK`&g0lA(!7soK&lX%9o-Ifn=3K!I%mdT)&lFsM&ujY6k1f0peXBNZ zw0&&}Rv$EYeX3`8UDyxyjWbWj-gtsGYz?0?$b|lTEUr20QRk44SK0>Kw+;BbS}flA zez3lsZ1H*ta|K+9K=hCQ0a0e-? zyklbNMQ|usMewM)^wW8pkAp`&e@SwNvKIwIMjxI+nrKrpVqo^D!nNwdY}*IY2WKNY zR3G%^@`&b5KlpBFPI#j1S8TeYyXz&Fm)brrMJ^}N-OOsb`-;lV2uk^Z~cNwmB%h2Dy6-dSWJW_tgg@gAWZ#h6eY0 zQ=#`CLmD(VGIDKHVS8>=VXgH~m~{9zpkFQEEvwl3Q|1)zrCfij#ZfEgKU#Wt3>XfV zo$$y;+L@!w^1qo=_&*jmf|(=_O*HN>cEi)epg<%1DrYdQq|L*<V9B!$TPzCeaK3k^Y(7WORzffO?}3Gd!BRE(y6}u*FyWv z$W-jN`!#=BYiVXJw46T#pU`>yIIA~*_R#qUYhE2be@M^xjSSr0roPS{O?@9BZJImR znzYp>zx+z?9FolM!k47;tG5q^?Qv!Qj(my@?O^>aoc~VhUWxAY-_@Ti{rC{+qRp;e z?ja%VIMUutntbh|xyj(gz}=DVd&d_z(?jh~A2`&JK4a*0)YFfC)bj~?Mn5XA{7eJz z&?#!~L@S#~J41QUo5k;GJdua!MDyItGqOhb##f8~E8X|Se-7!S|99D*jkKk>VU1Zs zN)w-4g6{gSws^goH50yz?zg$~LR&`_+*tkBdn_NNM-^n#b=t9-tg%{|4fP*mb*{$B zvk!$|-Q}^);x>anPB@Y;?Kxn6P~-36_mCKVm9I#?14$p=YTLRMzFra1Xhw9sBBX2Q zch{L51LS!CJSl#sxD;I7v<6>kSVrHzJexHi?1th}Y(?n1{mL<)5`F7TXyI)+c#|!p zyyA^(fmM_&#V#`PA;_%L-4Wsw<>>_8z=gn5o=eOfT0NftzZ%CP-^DBPKlh2Aa_=#= z#UawMyJ1_LpmNw2r|=)KEr4rmi}ZVT&$j3~uJCr-<__p|0DDU^{UGR7{ctq-m5IUn zbJY*%>^PfW=U_Wpd7q<|JZEELLpWNwt_K{QT%nc!z(=Ds7lWhIz#TS57xfnJz-GB4 zTe{^2`gvRUe7Bx&v3!xkeratkY*m^&XPUHc>3tJpy-E0n7p`W{4?2H<_l=^{{~tqw@GRh@DSseZl29OL^vma0cf9Lvu0UXkOye@iM(L9%>8onx0R| z7Q4{u^9xzmOw#AGs`dFdR1Q7=UH+e*Y3;~a$iVnmQ;+&=bh(~occdGzC+|GI@cmDC zx*R+I~3r zKQ7y|eLk@St=W>%>35z`_&9hJ4CNjE?C^%r?^cnsN-vRa{MWtMV2h=*(|!L(%TN8FU^PIe%*i7DTsMY6@IvKY z(_AGc*ToI*tvz7cH!(;%<-=}h4;bTRd`R$Q8+2**fGKU9bbV|OSZ^i1%)n<{4V_aT z`*ayM^b60TdEvH^v#C^vXiNNaJL4ibAv^qH>$88W;T`nENM}$$^KSoB=GiODd$X6o zThn4^Pnfn*kK*>VzF_=jv_boK^=@RI>AM$qnvcq}0k7|4k&V6&KZ%Bh{~?yg%AH#B z#phwQH;gZqj9EIQ_$BH$GRE2$F0$=iQSN7q(($bQ~!QY3+`N*L@ijB=- z%qNp@u9$9}w-gzh-Uc@|&{& z)b^=C=|_}PpNyQ+STH8cZ3^QX*yEV-_BgIK{i-bwWVdK6WuxdEzEJ;rJe)uyaaj`X zm+;CY(4Oj1z7W2*4o+StpAC;84@{r&LAkzX*t_DMC7&|mTGnarN?A_FU>Lcge5qu4 zE?&qWH(CGC`gldQ`*etG_s=z$r*VX)lFl`Fo4x__X#GnU(1$%a*I>ahBb{qdi5=*| zN9yiq4C72|pI?6qF@-u)*4QqG2c@Sj3frym*ZKADC6?KpU%!OB_0F%qneT6B%~Lp$ z%<$VGS&sdEmgx7D*V^;zp=&>GAW4^Pe|C6-V(fl{zc-@Kym66SRl76}k~t>M0sd4U z#Q)>9HpN)TE-L~@?NeNBM{Cg64t(*mFBANI-bOwEM{omZ=e@VW)7A#L0JzenVR?_% zdn>#ZDG$#s^va)It30&p$0Si-+1|XEGC_@ft&+QHW7#gL1Lv=R6UAp}*BD(7jCR_p zF^lYp&4qc4eCK<>{`p#UIex;2`t1ER>N0t>pAJ8({L5un>D{B{EjnA&qvd&ZX!%P9 z_qFe0V7%8a)4ItBw0tSDRPi|AqAo4_WB65bCygiXcC){0V*TBDVQt?1K6TdP-3)xt zr1i-W`uoP+= zcZjdUJ1+l2&#Z??_Q)@!J8*wYT0b$iMxRby(9K;2YkhxO%X>|8_1$y}6hE_mPT`@osrANh$(oK?-^d->{WI|9 zVBoLr@o{k`wwHY329I0uFD^Iw)!i4#^H-R0J~Jl*$R;rJfwPrZkAd3PTw z-`}rq`1(yd@$oCZSnEio2_WL>D&z^(J_}r;AeaY3&Ykh`s3H|9yj6Yp4 z%6QhZ=VLtvJkM-lopc61W8>TOaI7}mO&eCzAJ-@KSI>%Nak%Qs z(9T8B0Qo-yj%cTeA<*3_S-VE#+DuzY#nVMQ(A0eXMN>KcMN=m$Jsgj(3+8Dd%xi(E z`wpMVK~pCamvx%K#jX9PW51+B-*IBU{%&3&N59lw?PXD%c?a?3R{^gh9X_wqGj|-C z{`E3uJJNmsm$dWU-A1ITPaUj*b+PVc#s+?RhiKjN!C%xI+#S{vj*aOj_Oa<_tL^6u z+s~Q$KO^LwS@v1$W3$6&SI5n&!}{=Ln?Es|{?7srGvTQj@Kh^!z_(9b#)n>)kKXg< zLSS*>`ubP&tCw-qxV@QimYg^N|NU?-gWjmbeJgzq>82d(tN-+I%|WCO!#!9@YrtCD zlHAr_yUsN6_t?}O#OkxIFM2Gof2uS*KQ+(2ciH%?e)LwjNAEoLCT&ElOK=; zZ={U|Pr%Xqd=&eGG@p@gVu$FC^53_4oB38$8s$`<)2D>jO6x7OBVxbAbc(*emwdtMpdj54$(u{| zLmIyg{XKus%5PxEF8+i1NdINSGsiHrr}0r8%-h!wfCqhRTBEyyg7uu?VPZi{e((;&?LFH1^v7P$AYyy z-d_NXhVv1(N056i(Ba0n2hJ5^oe|wI7YAo1#t7cHcz$L*gRU>!>5c=-FwXOPc~EwGN`S#+G4 z7R)pMUYoj}f6v@&Qo$TEewG)Ct&FW=U|H|dStFA1YLonGf*IX6mo0tfWAHZo>aVXj zACb-%Gkyl>QRAR<&&+=_I85V@A+4(wxTYZKNCCZ|e#^j79o&WGu6q;Wq$Ysa)T;zvp-<9rpW$Iy%LqO!CrQZ}J&zd|-g zWr^4K1wmzBNsV#!&)S!4mArfv^kL+x=!kwC%zFgCHTqy+(Fc{$86M0rZdD)Tx2f{G z)_@WEs<`jX)alJ(6Z+m}@_6$W@;E<@=!G%v5si-FU;NUZ8e{JM{Ej=rh;Jqjd6h@J zZt`$%=xNT+%Qx9mqHpG)$bptiinkDhPae@I@Ova5%rmkefG2&Qk3Vk~E!6Vu>RT-u z1drwUb~fJUBs|U!<{3J)^i-UmohO;}QOZ`)R8o29z%Q@y)B0*v*%~w@S@D>K;jec- z2At2B_E)cm)Z^6volwS2LVML#jkD-C>1>sMYC>l4J%vxa5xUa(Wf7k8`p4~woeR_BYGCR`NSYFZ9i3xce zpKOO#jEw|-Bu7OnM%STZ%5V(!n?Yl2c#XQWzKZ;wmq7=#9~l(1_NMRChlkHn9`&o-hn#0)fqUsSonf*A z8{jeEO6SXOVsw#k4E&H+y}OS*I*?xN%;~eqU;D{K6Vv|0Tn7R9W0$ZxDM5$~v7O=DL- z#J;zMhd>MBWA>q1-Bf8`lD|{%Mxrml*BWjpA3S*se$=Nb-ez3ma#1kUrXp?AoP6%1 zvJrT`FT7tLi@~Ah z0pYrEOkaEDMaPClD6ciCM(hG3gUK(OKzE;1l@Wevhj3F*KDu^`7Jm9k@Cse&Sz{}h zy6@;)#OWD*&Aa5wdftV9Y@}Vf@2D0ZCyjSvUiFDSt6jDF0&S^K?iB5w z8zW1gdC7Ibm3)ra*1l}bhx)}u0eh1Row=3Nh&Udt~FCWaSy)^k5KKrZ6yw}(=X3PR}NA2hNX80Z) zb?~h56pW+bgFAQ+?L^NO>86sqdKOJZ_|$8Q;{(yZa4NnD z$2FM;lG>igpCh%=Yl9Cn#6!rJv^k_-)Bat!7d?S?oGtr^Cl}59%-!Wj9Y;~0S^tIx znl26RMBirjraCz$e<87E6Ge0E>ozg0De@|Qb|&wU{LPYofoyhj{*m|%Tp9VRbBMO` zF8T}c?a`FQU6o$)`$bvj2Qzd{J%&H5+&xr%p^YY{hW^~hekxy{`|I}(|Ni_*yd?Y^ z{)dJ%-(F*2cyy7k*1uK0ghuk%{zl|sZ$*~4_Br`}E_3IyT%*5|_e(UNrk(2Zl{`mv z#jdRlaCQtlu={HR*HB)vGQKt-f3z7B+UwQ^zG?IRo`2IP%K2*pJ4qLf49rAN*|mW* zK3CcIZf#(H-bLSv{o1*z#@c}B0^IxzpP^*a5BZPCr(8v+_;JUMZ)QIh)e{Z-`{9j! z?(Y}>p_g7tfB!p(VcOgN{(i0Bj^OXNYaWUI{wG9R;6?l)`Se@+?)?1+K{uj>hqdlY zT$7=*1b_eUEDZ|Q*nRBp-+0yD_4hybLbAXADbfwU!4nzjdw#r^er3`_eQ9h1zWJyZ9SpSgj#LIUl`&+#YY!@{P!b3k+Ys1K`?H1V5$^?X>j+I&8) zCTYy9Omgzr`2otZ3A~m0Nj9_Wad_R>2wIDR2YH^CQJjOn9%bQ*N6WBmT9sT(kVVeU zKG*B>Nb}QUh{uY|H)X@rnV-&%=Wh=4@3r}9@2<_W-eBbC?$5KXqlxDH^Q^Dg zy#K|&;i=d>>p9XzQ~o^brvH)ut1{2}Z}KG0vqqRZ-N%?ZK9-N|S+#|6QanIBR_b}a zm1Cvgfi$+~YCndFF^2Z;R$EnG<89z*?By4>=`KF0R;<)dsYm6E?*hC21^U|I*8^?f z|0>s$L4TL)65S-!W!gbL`8ilmYB*Nv?f7K&_T1^O!1+jXC$0O_=ES)ZXGBHjPG7L^ zZtnDL-Zgi6pZK-T-02STX{g9P~#!4+(wRh)E-OTxuVx_+1`2vhw6z!$iqifQk zPvoTOGqUhu()_v8Ik$&%rw4y;VZTVRQYVz>GM-J?+t@tlZ^~us&mG{lk%`e@Qt+qEUF0}%sAUs$)ruF@J5=eiYcAENKTl0BH$GqxV~!$Y#K)c4TmaMI@C zyuE+2SM9U=u+yv$u@-FFXJX2MP5CL{#H{H6Q?XjV-p;%A54$yqV_ZMTWBNgR)sKhC zYx+Svc;_bK%!o@blBO8duH2E`fwdoH>u|RUK2plrwdfA;p>-AMII|`NF8;ElvVkdU z#xi$acbCP9aN*X=1n;ZVBi~D<-<3PvBk|$%+r^;mF!)DzFg_|L9n#el+O)#!WZI}S zw|;jV?FooicX<`Jr|%irH{ecwAM@TEzAN98tmA2}(a{?8U257Gi=Cw3@i?PVjD6Mm z@NUx9ayM2UZ7m{gB-*-$JdU=O>Ul5F)~&yY(pJmws%h(T*WMUyz22pJw52^ZNwjql zwqu;O*zg;ntqY8O*%)mxS22FQMB2)QzHHfOcJ6OtJlmm(s=dGdSd(A)ec-niC#~Vx z2ca+Cvq8PNqj;-UEXOK`A7Hvzj=%HL#4Bfkf5V^9g5-$iW1?HlbIx;P13jO?vmeW` zl=o1UG#<;*3CyrBo=u#V{7G-#*&7)7*eD2RVU-Wces9EYgywBr%t7SUyiWB#ztz!C#V>jMe}{TD zF8dx7uH&&Wv9VFVC9@TicG-QFrlEVyD)QYpCVqMeUUMa^+{!}AYY{Y9(>~u0+w`+`u(@}TDd{F zOAS5m+8GM(_sTD;RX)~6pC@!qOBGM#ymdX=8259kkH%ks4@hHG`Cf@F5FT~UpY~{) zJ(=L~fKO^HEFH>LNu#r62caXhZmpPLLyrc>6&Yn@9cjidr9EZrd1(cf$qOzIxIF3) zadpe~GqRDol~=l3ar2k_7c^jFDwVd9cW`3%3w=RtG(24?vw`o=qW^vSl0H}B%g1>* z>aEa{##(gi>;~)#E3cAiFB1P^+Th3Hdu43hZX6vim&d_tgAa4B#9Sr7PFhnwiN03x ziq~#m#|E5NQ>*RbRqY*^@Dc8&$xZF<$}R0~ONHl}ckqnND#%vGhgnLETD@>Pv|z@X zZxKFCXtyuF@6jGR=sHrSk@n`R?LNtjQ_FkkJR2avZs+U<_IJzg!5$XQAKwh$XkVaz zKIYNjQaVO)L2nUWv7NL(S#fBg-#2*nRJ*TCa!+%V1$=Y-;qDnf-=>9oG2spI(p1iR z0+wvBG14JCi~fz>2tKYOJ(QOfJ6v}98~zratI_v$lnKY)?|X$WB=a3lOXsN1hPSO; z6uk~5mFz%` zrS=HvUOCyL)7-r)lEt)BIMuk6DC=aI_VN~aS6NqH{(aHeATV`~mgY4^PKp*Lg|zTA zWz1P+wB-q&cdqir9yxv2WsJS$4Rb~g{Mk00JzF_@E?^D1(7yO8x~P?xdaLrgl@yCCnOj8{z)7+vrfQwf^}*IvQ^v+aU*A`#Lh#zFzJc0ce=c|RL-&y0o4eRi9nvYNTR6Hgg z_0RGA;B&S8veIp8hvx3;d-YteVzb8OmK$T?@lfPi#CA)-&$xJ=1Y9KPM8zFf>BKr^ zjlYKReHG=B#xcSN3GI{~npUpUlf~ps^ou!p){IPfvCfq9`tRGuP8Vj_|MGyluL4{k zbIbB9)Jqi_IuyR~c|`Q!Vx}|jn0^mfzmVuue5d{8?)jDWxrOw0cv*fu^UOPX%H$V+ z`@SvBLB+?B@$vekHaI(DhTzN2q#x3EDdtszXLy+QoW^qmPA;@D%e)$cn{;mJ=`~;& zIo(3P=!^763u`Emv#o7iCLiNpGoR0=^ex%CTs`6C(x1BxY_%3dP z2CEq(+A4o_JI}H;ikpIh_NzM^wB6aD3Wz;gJQ`_XDWcAC;? zo3jtEG4Bo9huoo|vpHoS9-WWHx>}mv6`M-(C)6pPeE0COS8XB>ZX>e#_Ny!FH{lpo zWc3&Rt6Ek&JGei`ouxbN{GR7^^5H;3{@8tHg!0gm;UmGC=$>ORu1kO+nQCYN{*?aJ z+$&zUPj`*zm?MGVbWB?3DFN5$l^Qzc=Z{MsR@P6>qD+;Jv9#i3oahlbQjvw;Sou2W zDPYHS5d2uyLF%uM=g6ANUXu5ZaIOyRHhSUXg_LuL}|!Q+BUSNredgG)R7_k)8=x7l~e70n^rQfKb#gxgzblcQ z_3UlUAp-K95zpt1qv%=dA{u`^OSfhYgdduj8^LQTzrmER$e((2?dv}20m*sIC-j_2 zfkUewjJ-o$T6;aj@ht5%>#O2%(Gvgu+|JQ)g@(PgT+3hb32eeIWaJ@zl74(xGSK>c zWYcTiG?NORb+n**I!P0~n{hDjOF3s`39yz7oov&M?$zAi^Zk5)Z?v`S`!VZ;^y7V` zN9O*}&fS;py52p{z^RXW!-E#D8o$qJzcOv0k567~&r0rAnx}WqbFz#y+0%{sWgenj zWK4QT7z5g)^8_^xztK5iw9EK~!0irrQ}iL*!k^PCukQY7M!rRH=;59`JAr4$3*4S3 z+~z`?UwndJvmw9AbxD2|k3aUR4UqS2#~8haF|$jCCJ7hN*nr{*kj2KgCwr0cEsBN> z&ny|5O+KUN=&R!yGhf9vH1~!fOO$8ls*3G5oKyY#;Ts_~13np`9<48^TyZ%#2e#%E zvQK3jNghb{4&?X-jl7n$Fs6ff8GI`qt99PB^ro9*(Z0~90<7)O+h3S=c{Cl9b*hv5 zGiiIWUFKjrT?L(^gTuK~+}>FJbktUIz8~4+YKv?f&n1FhN8-&5Yu9b+*7UgW>ML3O5?FN2c` z-JP;|IM-B#H^1L;x|=Xk-(z&>>_hR{`LrQIk1<{w>E-dko zbiC$s!jIDUm+P03JwkI-*##W(y%%`>qv6aOEKu8WM7*B2){jBEwZnokH{>i=lU4Qf1q z(T?sMgU_OU+_yieHMW|#VN4ASV7d8!2ESXE=XdL6f5J#OR=eW1v5_mZB~rJ?<0|9( zkj%VWec>!KjhVBdoSu+>q)KKa;7_);=6&_%#tx53zFPSO!?N{dyy~gQPEXe7!uD${ zE4H4uZ^p<1h;eP74P{t@$Dr?V1m)z=D8^Tn&;7-SfAfA!!;pCOkeZ(zF z|DK|B_)>f3JHUBAGHRj9qJw4Q86M`$;rH@v##VXg&xd%|oWsa4XsM0%C^o^^j;+v& zY)8&@70gRWr~j7sDrXY(w?cm_D)x+QnLy`L=B%Dbpl8vq;Zx}OeBi2&5xS{S#?h|q z)ler@Y{obKx)L*f0pCZ0mqZKKt1aMRcgn6yDr;#&Z4FXBeN^OCJ=*-}2r%Va7-SqZ zk5RkJIL{}~Nngo517ZI>`mKR$jZY#^+df6;J)u77gos@8WdS_l)_qzIUj0ypJ9wSG zdHTuPYd=OOB1c4rk`0GZhwxXCSH5i!^3b-|ed3{@_s-|0?ce*{w%0YbZ}&#%Inwro z{{4MI9$%kJcZy!i_LFZr0k2-(K<8VoUVVHcwwCyK2L4v*%4@mLpnch)1+8hcKoiam z3(%d;-yV*C1y7FmjqNUe$FDNe`fr8fH~dv}9$1fF?@6N91l+03MpyzJOPK4kN6l#blo zj$G{6L>m5@W7SsdzyAhkZBV%7{z63rlHRr}u=pnvlYGfDX?uwr;(;Jmx&PGukx(l?7d-{7d zICXG3;Pp1`;&Frp*{+q&Q20e*XWLqX-5`5+QRs|v!5)i zA6WU3MxI2^Zt~ibW6XfLT6eVqrz~S*{dYF4(QUL@I5zT=x<=9`zCKKpxe0aoWj^vC zwjg~~zo#w3-^NetHRZJXNaOb?dnH_2u;%{br>*(e@tN14_g@p@Y}%SsA;s^tLzR4) z6Zn3D{hn$nr1*UoIKr)wZ*ND&{bjJ{eYAMicndb|FR?eiq;o^V`GJ+C;4{K%)FKIgI9-gXCz%9p013wr(0uns$csu z?|H=bR_S+(SJhp{MQ@aMlW14xgJq$i5MDJu7=BK9FdMALr`ux%pL#~(k0e{o*v5=e zq^xa&+P156f(&;)6w%4+p})PU*WU;{=^^R$<3HHA zUa!!$M+XUgjgM2KN-;f9VZ)SF6THZ20`;HC;wIxGq1jy_yc60 zbc4Q24&HEW<(qWnG-P3bENn*>wjc`wWT0fe$L{&r+y3ozqNaiX!!z=h5Lb>-IEwrC7L^~*n0`nZhP zPGa88_|k@p>^J$s?z@Ds&PU~+lSNveu8fmM`w8Q_SI)qVwZrN609hg(--K+his49T zi=#E?D-6qwOrOG==Fqzu;}M^y^cMF`oX3Y4-;wmqvUr|4_>mlOII4`Zho=!_(1BO& zT^TeHcuoc#F1p;4wl`4rF>KE&9hR@=g@0r}W<9(+yF@fvMV~Qv`#P_5LJEGVYDc=c z?}ZfyZQi+R0(-XhD_oQgo~hi4s=cH5!Z`iPwRQvo>%=# zw&&uJba^w{cB(^QCd{hQ+JfHcxupXsR!7CMFMcK0)k}eEZNs`F>pa`SH7*r94+r_s&&|Hovf9;pX9Vvst6?al@Io5nP3O zs)D+}1#5-Q|o;+w}9D%V0d`%DbE3AAo`?^IRtJ@0l z9c+Z=wI(Q;@-g=x+3zUH)>-C+^HCR|Aayoe2Y#rcWMC^^YosKJC}#FOlsS_ z%r}Eei<7wjhi~c&HcU@|Pc>re#^B3__wA?))W*0S6)O|>;kPHzvr%4 z;hs9i<}(T^%Uo!+&Wm}~u*7h+;u%nQ8JlOZMsJR7<;@5IDp?Yo)xnziZ7 zcU)Y|q2M-}9>&JtXN;wX@iE%dAI%@e!-!^+(hVI#^W``h`Wjy|b?fH46N~cK(7*JN z=4xjB($Ey+!ha<0As6yMW&WpjMDKS`ibeSyu{3`Su_)s&j@k)D_>-U6$2Z7-(!QeI ziADK8H-&yG`0+^>i*lXPz;Uo3FnfgB*6Tg2TRzi9V^P*qE;1&)Ba8vO=3-Ii{ok-% zjmDy!M*e!SC^^0x+Jqmzc2P`5B9q!0*yFRm_2bpz|3B7Plx4`pIL~OULVDfBq9{)z zu_*5#UpyA&RHcbN;K_j;b^zJTE`Cy)QDIzEwVU`Ew0VB7b#OhvL9CZVJ!n zP%Mjdq1tTLh>$VG)>Pqx)T43L{L`)HJ9+5jjQZFCUt^=1d-t>#O)y-!%yiCsqiz#- zW}c@EJ*B;4*o#JAV5jL0%4;dJKXsY=DB;Jec$Yp5+4dzufBin2x*gu|J_0nq0-NA)Gn7aWc)U6EhZC4-zf)k6wL( z{GEz3k?$-L&y=8BiwP61|z+us5XpjmToK|AS2rXT~ZwCjf|qkT_Rd+I$r`Mj;VL}!w# zq3rPd6gA2mVe-;W%3;ryWv}iA?;m~hVG&x*SIZs6`$p)HHme<~KeHb{_AZ&YH~PLU z&3ShAyYYX6>#|Li0J}wP13$u_WaEk8#joo^jd4P`s(HOvXNEe3b7PZeUlq^gHrRZO zgK59=7`oQK)}LJ;66!oW*0vjjC#6Xa=M;elWp(ITwmA8CNIY!9Tbg2`p)*f z8;8_C_a7fxy`%fNY5&;!T>sq0;*dTJJxAJ}Ah+L_kjJ-Gq zurZt~UyQ%3efr^h9EbEv;2GTsKOJ~M)Cb_~2Dc96^G{C_|0Li}WpoBc$ZOu(SB<Py630Znk~m6P1Jm7rnS?&I3(5W#~~@^%IMTh z*f{ik2Jg;hQ{2ok$`3#JcTUCQkXjuK)(~!cA7fHWw*t?PLwXWC%SMo095Ao^F7HO( zX&wrVzS+d}cyp217#Mqta(`DR{zo+7Y!ua@b&4|X>8~G$^a1O?&|0AGKYWyaMdWRb z`Y)(czxpOTI{QetTqZwq{}IIA;!BIgAN?!sQ=1!!KRRDNXyBA(U2nC#jPrt#n+@m@ z-*22KYZL17%e-q_b^OsV=gAqLD7-uEzHp6ZEBjl@X{?K+_`Ui@FP~ixc?J7Gg*)Lf zSo5vpgVk$~A8Bl4!^P!}Xu!qtJk;sd0Y~QZ3f^A2yhptp%VXwY;(IgSWqccrbG{S4 zN|NdI^kS8+t3&gzV$XUgC%yS;f*kZ?c@BgAG#`u8sPoB)R#{s!@;PjKMHWKa#@8)h zqM7forZNS1PB)&qym8%Fp>41234PsXvLsjWc}e!k-(X}3z6Rmqq^-yU^nvDOE|%xI zXTrWF#qxaG&J)cyUcC;gWoj9EiqgECy(x%Se=hTM8jRIhJVLCFWOJ1d zErF+AtUS_TCN^aau_+&CEq_^P4>cj96t~j*!{H5&5>GM`E$wR=u-|Yv-i_E=ijlk& znh5=nslpknLfQ+k^)CCF&cX|0bN=e#(C&IGX-PUcz7Djn``G`RUFah*P$#;+(BC(@ z{?fg=M%%}+4fIj8U~Sbq+wVK1JVy?er3y0~%mMaoxVnHp-NHQ+`?*mZe(Jj; z+q;prtU1_gOHeq$mOqeoRMi>dWj7aSQW^R;`rpI#Ma_9fS)0bEsm;Hr_Ioo=Gd}A9 z=w>9oGvB@{@w~2XG+3!En983Pa-HIWN>7_S3vTyX# z&zFs?m%iUX9`Rzhh7pVNvA8PZ$-|x7w~_R%oR&|vZz?*k_Poo{tXT&Qg1e1=f)-M& z_cgIzv!*P=YQqqXs&9$)d3`Cjx3@CaPF&Y(=Db1M7PT+n{byrCIV0XkqBBSDW{oE0 ztDZduunnUfc+_bJCq4AG+VJ@3Yn?|K`qK-l18Yz0I2Et^K6WgZ|CD zf@l9ukoUWE*6Q7zAZL@7!Ji(#6XXv+6Tf3`W4B7C=}wU4Y{?4Bs=f%WeR~O92^Tu^eWZIt9-z(gJ*3uf4>>sd1@FNg z+#}Lv`XxH3%<1AbzM}`>PU|p1z55HmwPf;IjjQn78qOWV{@17b|0Mo0SRE8bl9x`V zi*5{`#Bpo+#I@Vdv41X^_gy@ZfyT8?*e+Tp{x0%QWeYd#v5%`LBG3_|eoJv&RHEFoxe)+V-~8*eBQFqdaZgcRo3Wf4#T2 zj9F*)c=CPPePKD_PqOC--o=;ls~*a;;Ya!}*y4ZT>1%2m@0vHOzP9Ed)ThXvaQ++= z9wd)Dn{(Q9Y*xzM@2ByuSm>=5mda^;+Q3T%r4{#D`lOz3lE=q!oc=YoYX3IDf(9b= ziCl@>iB&d&rMDm$y0J_nInUQ%W#x8aXLdjX#y3rU#dM1~Q*8b?p20=i_)wm81;LjM zeb5%kVa0xm#)I);JcZ6a5bN^}UVpc>)=mVZ7Q}f|Bs8v|Evx zCXISSSy%Br__EHxr=7x&#?k0w)yK1F+1MhqJ0kabEBZ?E)UBndZHh;^9UWDRS0u z4LX!gX_Ur9Uxt?`qkgxy2(K+qZu8C>ZhVZXbDO`qn~apzia!<`rRa7 z2CY?}ZC}I>l8fbWNN9U`|64O{(HK+4wdW|?9^qL16O9`l8y}vL#5`!C`}P`*4dtDT zP`wrm+w%$a9I{ z(f@j}&+oNmg|AhnU)8cQ?)#QM?nv4b!uI510*$z`#@?pfG|ENBGheOCr1z9Bgderv zoNN(X@ww@|zhHsHdDSSDF7R?yZG0%SDrA ze(>qg(4h8{h3EKuTI(U`Rm}qiMPu;i#dd$u7wlTeJ@=AFa*RGG&)XzZTfY5C<=tfS ziasUd&=(UWH%}pt&W7pFG5*k5=klQM_*8iBmGX%Os&uE<&Ozu&>m~i^rdvB(gT76a z*-YD|pGAkV1*FHs%i>qbJH<-A3c4^n!@JKThQ>*g-ZkqmoZ0%%c0F8U^Z0w&|A zJ$Wcco7KLK^t09>cJd!Zvg$>6pte-3frhYx8F*1*o;8>ZhdztF~agLmrbOfMeV$hSx7 zw@KsqH7d`yqg?%W+4^&bc6a3uo!j25xg`6W(wSFxr6&z#-WWXlz$=5ZZcCpt#97Wa zyshsIx@+pdQ?IzK8`@YGq^3UHkv=Tk|J-|Mx7sq1I$o7x>`h-6)5e48GynO6hf>tr zl|OsP^queGhpzM?L!Y3H9r;5{zc%LIN!#BE-p(ExOMBFIVk`I`ydx-mlXAs7;3+%yVSf)CT58}OXyKg!yfcRSZ{sXO3lH1A+`i(c zEWDe8(ia8q<|sVQJ}jL>djBn<&f0zpvU(Ht#HZLlYT>bmvjT6Og?D{W`n2F(AA@&O zPa!h3$N3Mx5~o1n)5mY@9G%5Jom2(-dYQ9E%5TdTa_)%xA3m> z;ay_ktq4k&3Eqkryp=&|f5E%l!n+)JD}i?jvB?(R6+XPh7T(gJv`p}p#^5avO8W`k zg%;k0z*`Qy#he9e;a%XvJI%s7Jt)0h@J^4xTNIQYS`0k)h>CW%p98!_z&kBl`k;k( zrVsBp3vYfDz*LvW0gt@N&RA4jpOX5ma9p-WHn-qfHpUhSJ2gV!3A?x0-%3=3}t@LGX)c((K}7T&== zys;ME0YPae?JzpW;}7khxL5EdS$LCxHvxEKv!yNzPk7ZH8ei^8S23qQOW*FKoycY5 zM|okzH@dbF8{UzAw>j6&>_Z(L!j?{1MjP6KAovKs2l!0}z(xGJ`DL1d;9dN>_zm%+ zzTltvAQr<_`<4a9=b}Awy5F~hvrs$pM+|KT zr}EcnPpkH0s84#&^4yt!58tz;oR=>9(8T3~yMAzWCEtWs;SE|dzJ!nRFB`cQ&X$9- zovSKmq3XOXqeo|Hk7&^M*S0>5tL$S?Fwxn`!s7wpu@*k^BgB-d{l>?{7|Y(0J-STo zPRHiXxzG-hZVp`dq$5X9JnD#DM*XU{cnfXZUlB7GNT)OFFAab(i6L zc1vhqU{8Zz$#pY!*e&w)$`9MlZ?M(u8$KvQjC0q@DRcK%|H*IW<=BVjt`2-TW3c12 zP6C|5t5boo>6CY{H7om%@-1UjzH8-@xn1rYP~g0G*sYU!ehJ3N*(4elgCqPuY2ZcZ zFqTjF5YGQ#W9}k(dn@*Le2h9r2Yu`@E89x4bu}NINE-ZMVq$dmQ^daZ;Q77}wO2N| zIakz-k7vin+RHqo>~BG?MEh0;emy^_a8MrW8Zczj%Vt!cg-SghpcHBU~OG&*Slr`X^qIeed{Fu``zw(=EIjxw!1e$a<$}O*oSr6#Q57tp6N* zVCJRhf*sNkrw0Yi-(5a`eh~U;!u$0yz)@XF8)Oc`c^hWk-!I$q?n)fePtMgGyTS`e zw0pj7Q-t1QxN$rw*%rZJ0&LAGcR=f!L%I3t4rG#i2KDL?Uc}E!?K~z@Ppmz=lh?&r z-vWL{f<4l{QRAbatzWU@*r7OV`u$t{07d#5Aio~qn`H4S`eXbS^iO`3512WbCnpkc zXy5~{ao(@QilZmXXL4ws*gpdq)yf(bcF+####ZEk^cMC|KWF{E($Ssd1UfR4;vXAk zyX+UfuZD&UUGm)WgFkTYhtb95I(++OwZU!Wyh+X9jV_+b{$9$L9ev+iQ#I{25YH~a69@8-WVC}j2zWq2ao8uCr@RK5iDz0>&_fG>RO92n`$ z=kWvfTi>Ey<9Hqylo%5uFV3Xx;t9cftLq0erG1De zI^Uqyl5TiQ;l8Hend$c*RygX=)eCo!w!Hf-g&{X~ls{JGX@~To+CGgo=zlu(H}NbT zW_+(aKLIVcwFNWwdGmM>Z72#!XynO$Z@aJR^GIV_?%R@hSc!z`656|9Jyk`9T|KEbgdguR7 zX3ax(;E3n{lP~G~|5qzrc8J=X5B;k9lUCj4ip}fe-tV{Mu<&f!0}hW$gyZ$S#u)4< z`#__9#JJr?yA$=`$8Gt8pvTZQ_Ta>L*U7%6Dgz!rg0D~VQ!+Lw?&r22RLCY^hYXr`&H(X30KFlAHI)CK@e0l^{2JqxPNSpHG~VTK{c+PWMUY ze;xg8-X-4)*eO9KdapwRZFxlVty*nqp!_<`DQlIl(U)`S$A(y6uH6fLIcC2)ZE2t{ z|52}ejlLXBKi(JX%RBc%UslNXw2OF6(~bqPw#?oOZF$A0U2jYOJ5%eY#M*M;UTDi5 zyU~^{wB>*J`MmQqw(Fkq&P6)kY!}DqleFU}v9^4BFSMm4v+HfSoOXOU)|OlLLR)U% zjkcUcJ8q1%YTn_M^Ww^wceuOh6!#E65A4GZKDN?8E?mbxFA=`yXZ7@% zk5|nRG{=*l(5&?{r)A!HmP^ZEtKc)dw8*(PVeDBjPkBw;1U7~I9U0;?=x<4}Fj`l~ zrY?+6Y)}2U#zmBg*odAEHFA?W++0TW= zU~ui%aa|wtjuG0Yy_b!&Pd0CrZ`FhC+9ltb^F6uwtnphI+uHLf<*}`yNjGN^@0c-5 zF;0%9p}r&@>Y<%4HO(KjmuP-D zaQ=vB{)#Fc-4g2P7n|l=cZ25NP>1I4_+B#2|EcJqBtH02O!k{NM%}yF{Nn3?CHPzG z>3}}@W05f<`L}N-UlKnYDn5zn0L63et`6}1+avq-qc(QEJ8Knp-q4-3=zHb_FV3B{ z?2Tw5*82O4Y-}WUNW@0)(8 zGNB!>czZYZn*Qe@*XRAmpTAtMd7tu&zz@a$**pBu{>Q}+HGL6(>UPd$-~IFFhnHfG zd-Pu7gA0N4N2C}2Z0-HK{{7FN4_-=p|Gk_ESkF#+AMq89*!!D+lf(~yOZm9HZ(=;_ z`NC`H!+1Ok_edr9jyxap2==i2&58DK?$2YM@NZ}QiO&izcm=2C3|p;3w0s3bwUQF>&Cj;L2Kl*gah<-PN^+z3qp6`?vr6`QoLt z%Uu9IHPW@8aOMm;@`g-BS2KB;Mfs@1orzYqH?d7hk z7h>~e#jd&Yu#8?{zn9xD6}Y>0NppH(V79q;a*O8Tb5%dzOnjO4k11wHYuBuI{95`U zPkb4%U&--hUc6hrI-W`AIqpil#--#>^bgdTORQFz8Zl#m_#mOISrbP8DL$$HxXS$! zoLO)MYnoToTH|wbViOZ);)-jWtLVvQjibAts7M*@=hEIYjm6=#eVe^c#Mrk(*Dq{K z1y_LU@GfGTk66bo<}Tzb4_uUCzxNZv*DoBC8Z*|Ung5{2eOGzWfZ7<@@0Bw#YFp3` zTQb4XQ(B(6V`1hBvv>TG!3{SbrakHShhJTeAxjv~m)HDRhi5Gw8cE0$p(evz==DT3L4~o)0j+=ZruwqMK;5($dtE{e^+IS z;*o2~lqdhKTBda7rgl4-(3xA>ow&C(CWrho9l51n*E}}$STHNGtaumM!EXz(wObT( z#&1gy@SNs1EFidn0uTZg^JJC95VDv=~U8_&JE)XB!j;%`|VpN z8;5hvt`ADr3*PlHcsFsbso-r0Ac zO%}Yh7T#Lm<$Gx@eo&ezc=KcMa_mzTypt`wlYy55-f`K| zGZx+nKD?P0-mGvOi?d?zW(TFu3EmtFZw~Ng18-)w^pu5H>)f!k+BY=@uQe!jA~$rd zhH$lg2Jl*e$JrT03y<@CD&sNMw(kJ;$IuSh+7Wu55R~o|yh#?`B;ZW|-dOI{w(wMs z&JFYBu7B>vM%vlvxnYZFLmP_vBm5rVw@c@S`SS?o4M}^IzeSs68zk;kZlHV)9@&%f zTlTR0oITv%oUXUOxq<#)xrgO{w}<5~t+%JSf%ZSShvhGf4 zed#Y=jwjruJ@uZwxUcNJg~S}xlgrJ|H^y&kfRiMb_osZ^-rINjd$rTw;kw@l`($LAvb{_hE zZ^mNX{jK`)p6l|RbHD=pd8`0khOl*a9p&k)a0&rWF~#-x$?<9C6RBv)D#qr!Tk zk%f6;Kv;j7xflIW+dO}iqZiGif)p`R@NeYYw1l|lmuaPOTA45ct(?#xt(;hcR^FOG zD>{#9Z}~3Ev5?ROaZ6!5@MotTKuquwOEWcNf=w(WzG{5eicv^tlZQud{%_XS{uleZ z|Nfix<;!0;NHbpsPLh1NVs~gJ7u|2Vi}ABd{P@|6R6py-BgM}q(@e6@=zlxSlK zdFr^p+>0Yy%X~ZVwNrxW!Iof6P!qnZ_2PtU(yIrfT)xq?{EN#MsZ6#s+R}i|I<2Df zh#U`Nsw5xGxu~22t=KPf|K5JV-R0*L>o&|=bPw4q!6dd!ck?N}So0d40ic*;<(G_@ zXX*{#2jDaDw|w~KJNdKr*_><5yY8uaD`~PLy4Ww$&%PT!X80ZVB=6_ac>O<9SB#hK z*oe+0AIf8vFc#`}Jvo3Z>2t`=dim7HSH)rNLOY<%XghjEH%*&&GH3MH(A-*? z{4&yS5gBajQ(Nlw`RWGxvP-;BqphL^&7GUL?=RdtEX4E#2}wFE6sa-Vk3`Q`QbbBCtzU3G7v?yfX2 zxf8B9>E^?f&+z4^vOPs$n&+FdJ*T8Ye(eDE0hH|mw#p1xnEkX}>HVjPm(Lyg7tU$D zo1dZG)3T+F;AI@&v6Bqnv)N~&YrpHychM6{(@*fso(%5&-Ne3-&k}>*)!Ni282WY| z|AzK?R{0Ef2n4Ct&-7m(Z0X3q`-M#K{?C~Ab-|W*k#}HSc1!>D*)2^$cFXk4QB#6x z)2G~S@7~jxWJU*HR=t~fK1FA@a_@YRm~UVWT{DI8A@1AQ%+$x7wF}Mt^r`HP#8I!? zoNMYEBpzLN$Iq$NuFHchbeuT+EFs41ahg1R%Ey7*PyBi&nD?2^OHu-1ALuAIqrL~_&j%L z17-VXhwzKDL)Xn{7T_yDG2*N@~|vDM;aMmUcUO}#?t znp*&45SZ|SfnA(Q+&gr3x$2)~>z_%USz#W*%}`b}UvzY=x`=C6y7m?rT2g%b;mlQd z56JsL{zb=Q!GZZsy-QTb)Ub|0;_r1YrP?9(Wa=)qWCe@y0`Cz-22Yp3oN%!(@Oivlw zglzvTvR!uZa-AuSjMqM5%Je{k_Fn5F=5`>HbVvE;saN}yWk2hFa@|qd4=pv3_ltb% z7p&@PkYJqYsyiNTa$gET%T9g@9%VvC;W8! zRj)5S_r!1*>#yT|+0SP%_7QydCbWHiFwbv0`>*DS-Vw%jZtAr^sXX-TmlwYlcB8Cl zP&TjZ^e-iquZ5!oTicyA@tAF!yKi;}WBM5N+-LCO+0{O5e_Rs!tG>FuyZZwtl#7)+ z)Q|bvKDt9aHs?CNyY?OG<7t=ZnYoFX?@zYxk~?aDWd3FQ-v4~+OWCsro~uQBK3(lV zc0}kZAC(K@^&cnXDaR;5bD9^J^-tYD1kD+FjI1l)Ka_xb`DC4*kgvKWg`TIbX{l`| z1Z!3wADra-$~?MFXkX3wGkMGVv*Bm658BFXjkWY2^TBx;Y=ZV9gF*}UV`;uSeR%jR zBlqdoNPUR&OJ!c_wYf(B{j$!FWN47PtRr$N7s_@wZn6c?7Zur(^VgWd`Ta)p<_`4T zW3r<(FGbH8yNdovU!A;7vd3E!N`PU;hw{cpMXYeAwaHI`ujM;}XN>=9Gw|CLOPp%? zhX?GuT6OMtGkc-QCtNyw{?O>|DMNy(eUIAf$R6P`zQx}8cYCHf7_TYpZ#Q#9*)m>Vv4DwVaj+*ouJ1e>zXi0iiub30 zX=nxdGUKk;@mGX!w_=B>d{=tOkj7kN_g**01WqUE?`M4F`*r!mTV0fUp82C}ea&5- zGBj``bLaDbq565oW-5+xbMNzp&LfZd@WMnpwrVFfpUQ}jWz(HT9?c8H3&uWcRh!Qv zPYBP!4%>3x&;xd!-a(#|Ed0lSFI!UO2V2D>m3!YEEY>w93Qwx@wUpDiK9CCMqy~R# z6SSo7N2^WHf@za$3+b}yklhB3=__*}?LSreDy;Oof*9lSZebZC%z+h~h$FS<~jwftza zp&_*;qMPzjz2anFNdOP|Y_|_&>)%#}WA%t6Ms)$_MQC8=iIk zJm2@V@GG_TwEUsIo}Ogig@;;tItX64u@N2p`*|bl=wCOGr>^fb0q17i({|h$f6=uW zckowq{!_+J^dp^8@t=CS&&O*%lwnStwRQ8(bWjL5S0&hgL1{Yw(}Q&3#GS+6N6!^LVaDygu5Ib14`XYT*L9~U zf6!+95pTW`{|PX;d+*NRjM3$?SH{LXxvu@n`7ibAisXGJYunXa&inp^y!Ze&{FvXb z_;H>m_Cs~MH78d8MQgvFFxTZ+6%$-lXmJ(RmN}UwC3#2{t5iI@IQ(Far__7|9JkV@_#V@t-(&8a z@nF{k{3`UfqjzdI_tkdGRtafxRN<`|`RSlDUYrmC(gUb`kq-knMhiX<^pY-!|y>VRhY$G zY&dQ4kv89Z7>`RnU~wWi%7uAg-=A!Xa2?jzAkdxG76vas5BCyls`17yy8vIOaHa9yLBGGz6x?}YrnT_c z)}Zk8;nESnl1+$>*1v;t*a66}x!`SH>k+~Fhs6ijV)bb6({nj%1bbd`cju~;HxH+q zEDcus(z<4cezcApW4*jG)(?)Ro!Efbg$pxff7Fz@!#6<#*dC4s%=u*2Un_fM^QNFr z>u9pETps^UZG4sQf^%*!V#Z(9t1T`QN&?Fs^x z8sHhq-%A-itK7#Z=lkwthyLnK@I`OM-Vt6D3*^>wKRPw+llZcuHR$^~@1Z^6$&Y+R zZ-`&v0V9J|<}va|=sZ@J>Umsss2p^wIyN;>hiLqA==s&q>mk!duHzu`_;nn{dsROY z+Lk{!T5kWOa&`KDaJ+4?yz0kzY>Dwh8k>>z)JG9=B2lKX&_K!4u`5}1>Eh7i# zif&`L^y%)ZINina3h!Tw=V<_spNiq}^$p-Lmw?AUm1_WxC&uwOy{T~Rz2Ued;c;C9 zc>Jh`$6viMhR0TH7x;HNG$6WJN?u0~=Ijz^@2^RV&|d;>FRsF^*2)iJt$YGHZofit zc1UYoE71>gpx4|K>;cwSpfgu@mU;%}7+qlKagO{T$65J08##I$@{zf7M2FmU_}st3 zUX^ZK!JG`5xV4|WORcPx?$o-w)<>tEqkQ37>1*_yKFs(Rbe3ReKGk>9x%B-&t2aoy zjCbbVJ)Jk9V|owm)|@?978I_7uFLC|v(Qn!slLG(^pAB{muJxVcV@s#J1C!@JoKFU zbB^VOb67{;FRaUyNi`Kz&nns_J{>@h7@lbj=i{66ljz4}@RjPjQuV|0`mVmp&msNt z|GzxHs8YfQ-#MqPycN!j^K9AW*s(IeNb0! zQuo^|J~_X+od5l_L-=8>+u$aHJ)~GN2k$1}4dj{%iUa6h-W2N63E9%X>8#nS57s}# zx>>1svha6Kx7ODj4uy}a{|S7c{|zo!U(?yo%n5t6?xyccZJ&J{zFIhhrlr@)ID-G> ze;u43u#O&-I&)2ZXV|jJBN`Zf*YJib(Zy}Pj)lIu?pj>YvEb#GU$8nB+P$q=ys?Ni z+J)Ffc8!+w&yjB6rr$HvBAz47#eBte?&T9C<7qGL>u3%7R!m9IRr#pSl{~u|9db4C z-V^SQ@nd6fxzc3m4Y>5j$&uHB`+l|2AIImX%$+^2BHKiJ>Zg2#qGQu% z*fiK=+yP+Z6Ok7xt0{U-l?XdMwjYu`k~zTK06GY)hRJnOT#<7a7W)3zTOT zat}H5VLi{Zw7DER6I+mT7ZrOM!bdM&)EvU^=eyx|#!7XoU&hXyAwHvy)assrGg-bLl-xCPBX>miTJj6u zPwh9nLFW@HZwGi{PG!p773U#v-}Ss_rvmS9ZJmPipTOC<>NNN#9mY7o+s7YU%`>tc ze|IE!f=-?7IFa`%yDneND@E+aPhm?%^jL3&_B4Ki|N9vH%MVO|pHOZnR_@}Yanl?6AKa<}``6Nr;N&QuJV**U&y+dt>-+j1|b9#FZ?0X7-$Wym5 zPt0ZKbs>juLk{1{IF7?^`23ayrO&6brOzWrZbwhuM(on9#6Kzis55`?(8lz^Lz{Xh zc7HZMvHNrBi9`R|JE8lQ{Dkh!D<{nDN>AwSNS{5l8eeDs9q>o~?4eD}VGKXsAs)^= z8KgrWNYJ$Up`St*I4SNe%(6dRu zIlHqXKmLU-;&F(REA{W#zwk+w`xLV9<^`o4n--M1cBK0hk2jt9?z-7wTRKR)!+YMz zvpX-gxB=e&RH~#r8-zn10530WFZF{b#o$I`I@@(3(3XH4_i(n{N3r7u>$uD1QU8r_f^TPR*aJ+$v! z+YaSh|e32 zW@JX7sgi&SBX%Hzn%0a%m2oKTGqy8TNl07HC9SmL2-jNkyg%#gbF*)9!;g+W-^cIx z$9|D?GmOZhTuo{L%jAPrY$Z*fcEoq4O8|_royJYjour zDYx9C$Gg#6PW*iOAvG-cj(F8v8vMhJJR3@b*lw*(Mg7p9{?9lc)49yiSteW_ecp}! zQsQSMU!8=EXuh}Q$)bb$U)jzE$Lj`UXc~N;R>!*9f<4u4&qU#_4RzVW8|$*IjSFUd zDYalv*qsi#X)kA3$2%!6>A7Y)|q$;IRi z7s_rXdWzisZ`e!veC!SRI-&hOXz1SJH0$yzqxj+fp}p7DZQ%l!kGgJDz6JUDipYmP zXcJA=64!;`5HCV#q}jI&!Sthka(;6%U(Voh44M>LAUh~Hw0gz^KSLj6TdHk zJ{c1d$PxntvUK(6mSIpdsJ>O#aj66$u5=PEnK3c*#@6@nu z=}*+IuNt9v?uH&syP*3~=qapxUmY6RVC&D#m*)DjPW{=Q$cF96@Z`kH)8h4U!oxnX z`iasJ-~YnYPYm0ry^OntM0?n2Yi}K603J@GPIPRDZ*1$`GoXbo(i%YvdMccatdRbB z>9SRkuIQY`9Ap)1NwaISJu|WMGitLhPS^Y{5pXs-dl~JVMjO>0wN33=QnYGIB9Y*`oHegF7hg0qPW)L6UCIDPdsTB&mNTWCrXK* zK|E;|&z_XUk05?u?WrsM;YNvrbaVd8Um=$dNQ5ocQ4Df21v**p5?`FF#A`e8V9JF!7Ktv`kbL%xjP zLZ7T8ubtx&Kk+-ocYmMkBlKBs$0+tG>X?&%$M!j8Yae%Xf8qL1wExESk$jIAzU2Cg zI{v`n+1zx)NzD^ev>{%r&>H)>r2F&vLi5(^q(@5n&09PA<{LkUhViTtyRF2`L-s0}zo!yeqmio3E`-+U&KD^E7gQ(t2leBdh8+x-YD~~qOF7E{52wPzsqnBB z-<|QZt(Em5&0n`LZ+(!pvS!xE9y)%(tS>cPu;e-Mw_`w@^riL8+0MFhX$+lkfuJVRY3IpF>peGKG zigb;>E6EU_zUS^eU5!J7a8t5PWxr0@&c)c;23NL$vA+s+F2|)c^90kdqZ=*ZKD6_^q>= zn{M5+g}&5z<457S;cgld+;u;4bsu#=FI)e7Cl{icUJ;0D8w$!3|)D?FM_1 zI`i44{|d2ofQ_>{c{0}t?mwfDYq<NGX zI6KADcve!IT{is`b}GdhyOLAGU7?Nwav?G82r^Jp0v;zRs^`4bRr~p!0w*fj_TIeYB?C1@4eE*?q0& z=Y3C<>N`^*+|k-LbdfQg+tS{|`#*wSL_WdYbN7(Q&gi_ArGw+=A0^$|(swSu-^VYY-XwsyWPzX-hYbn%B%Is2Y9!>Ani%s?YkPrkJ%c=gv$JYc*(oJ z52~}6zj{M{-^F~FuaKbaZyoQxbE2MUyjwrP!#tjOe9kLh%y)}k`3ZxSZ^~as74OwA zYWv5}(KmOA8+>0QKIf+X5ysqkAlG*)i=NRuLHA7crF;FR_T~E}oV%)^-i}0Wp7WMo zHwT1Xm3$cenCA?uyN0<$;@~#c?k}Z{+wcc2E#-+%aOntzC!U--nla9vhwW3jGs<1a z!{KqY_I{{y{PBG0 zGWHXBKdN`uZMX2hUJo`cJh{1>y-&B6zqzUC;PC5_>E@y!nPLvb9`O%<9q~Uw*w&iL zlL!BWeJI~HgmdY$xvUL!+_vY7q*=S0qi^C?eU33kn$Cx-F3INq!tNtqb`MP#yu4Ro zc-O7IQ9S+Ay6d)CU#h=tPgtL{wW%+IZEJVp;2id5j6U%igMZ3g!Td7hBb>tAFAi4_ zu0-!mfo5fGcCKrq&hosp6@3i82f(JbK1>_c_V3JrH(=Za9_i&!^SuJ|jvGxLZeq@H zBWLuYeL#H&ZZfec(WZS8KdiZwA2!|K6koE>(u}cEoXpjB_^8JiU&@q50HY#*38||LI!1pU`-mgxX-}TL(U!iYE{@{!0#A8o{qr;yi zlS{vtIDg4R#*5Bwyt=tEaZqQWPerc~KU79M&BIWjuM$bt0BGA_yoqgh{B22pn*WAh zD6G4h(}N-m+JaWJY_VteoR<2^RPD!?i-C^KPb;HD!@PO zr%3+eee8cagE};>#z9;ETQX`u9Xi`FlzT~%cBhbh-v`-uZ*H2nM|H-b3DF8?OC)oexN0`zN_I48^^Xu;z+gyO>|$!%8P6mL!8B@F<>n z*J$*z`Toj(DfwU2yq!8rIBz7Ms_Ith|HQhgTk*@A-nWPM4}aFJFN?5?|W0 zzu2t>xq8Yu^U{eQFIyPbeIaqI1?dbv^Uy@_dcvFIbGn2jH)Y_{`j+INigk^5YkpVy z{qa@B%r}L7NNwA_$;6dDUCk5ZkE)M`l1F1ua|8K~wuTSwNz|n>Kc)es$$L zxwFBY)lYfnRaj@TJ6{)YzOrND9^Eyd{?NIy6ZlCj@J#Ey>XSJBX6_v{e};L%mX7FL z=3?SHSe3 z7z^#&vLJa-_mK|!4E+UfB`+}swZFU+I?ybp{`h>1!lMc6EVkBE6XdYqdnHzXUI+Jhfyx`w%x{7320 z2jPk6Jp_#qUdzwdnPZI`;n29s&=!sFN$zZi{4e=*$T{cCYO}LAV~2lZ^$&j) zjkCCXnzLr&o2@_5k3IC8#=k!Y`y%yY)2wY9dH2Vp0cpT80HEC_2%wzBA?Npz#1B~?VU4h zEwPTI?B7w=_WC$)Qy0q=?R}H^h*AAh_z&=__*uWuU+R~QlzYVKA${X3d#U>Xwf+^i zDV>bk_?R18idP?BPCogi3ip>a-(S>x!}otAXA!P`{zQE=&oEz}F`M%f{P;Cqd`I7$ zrjUQky4*njxXH8olGVyYpnE5;;^Q*EL+Pa181U2mx@3nO z9r-2lr|`)OuD>Xs(Mm>ZWK(%+taWmymtS{0_shlgtIzmI+18%k4gbUso0sp`xb{h3 z;-~bVzH55W`>v_ldZYkqmx+sAz@z-+te`lcf)j{t%e%EjwHLs7ckFTywaTZm&E4gh8 zJh6K&rHjCgE-)FnMdw)QH~A9PdS^_lZJR3-2j53Hl7H`Bk&ZLJYi>HYbl2Prp44ae zZRy?-Zl3o%Ae}4IrNs@(RDF(pY2_9<0)I{t~tfW zcg?+lcQ#|9JD}!jFaC~`=M$#SZI7ni=_TEjC%vS0p2=NvQ&h(SzCq#{^GxN_>@y5@ z*W5>h<1}~8U0dj`xf=O@x1D@(7-PWg*Qsv4*UV22b^WaF>z&bb{! zXNBz;dg-TAhevf3jQE&ac`csSUdbCT!0JLjoD%0ygTRc9aaBo)j!SEKZWwsqOzhpiLz#={X6H-A&MWx z552lgcB>12=3K%F#);NRwg3G=cwp(YbE)dU_N$EA zVeLV0C_j6f1Mi%}4p@9V&XM0IsL#rwTdJH)s=WLJ-8t7wy6Vo|IhVrkwf3NY(D}M^ z?i|8{-Z|HF^U0^H@cYQ?-vb`ohxbDNR(?#1@^xds{L-VpWSufkf7TCTv-_2Ww>cY% zJhnZ@Ts4nQzx=8z1k;QB^76U*M9;U8dHYB{ree>w`SGXmBM#F~nhzq2wytmx93~&B zXiefj48OO&S^)nK2ElH8$5+nT+>U|Y;cAX~syprWLi==g+U+5a_Fd&y>kbgjkz`Y~ z=l)I76QyQjUf$9T6r z+27Kgc9SIEe*d0{|Jbk3Vi}cxhEtd?cm488K5wE;g=8}xSGoT6aU~1OV=3|IfHS_+ zPG=Ow@4WguPVEo(s0_IC*1-ytk?hN#(YlL#9Ie4Uo2$>0dB0tgJ$d%y{q{EH>mOnH zMr%Jk;L0kkApiVv(~q9tf6`~C<>s{deCtzIc9WKOTzjpX58oJl^^B`~6&iCz2amV?%~?Cx$Sm zwO8+a_J%LxpCD7oGrV_alHv8fv-aCed$rf7b(qqAcjPrbH2B>+qoFB&Y3)c@I0I#2 z>9GNRyleZ}+>tjPU6jYa0z5vaKpyF7TLb_79eH`WM|0%?W1@iU^&gMg&r&&)!T$?) zI^O$!z*H+>tka%D_AF{$AfZvVNy=@LKQ4`wn+a+!?RQ z;=f)VfB(#Tg8f7Pj=anC9kh2x-ak`TcjPV8d<363z9a9#!Dw(t-UQL`?#O%C(K!72 z=!}xaqVz~t6Z3L>md%XAoA@7io7*z({kHxl@5uYS>4J!t=&b>F0TtEJXZsSR6KV_i69QJN~P__KG+96}lrYqkPz_-}oJQ zu?>NC&ZqJ17`8~ZTWeisb8jAJQp`8^?^NnP$Mb!nKO%pgT{=Tt<_lua{}rF#=0LO~ zw!8XY>=Y4H06d8zlueEjG zhvH21X?>_Oy}$7LuRA>tzkLRIG=Gs#-RD1eeq;fj+xVkQ74oZnW9gi#WVFr@{)X-^ zyw=%R)3L4|?N?=#Pk%ZHKwp|b;y zqfS*CIQNq`nrHRxP5b+Wh4%9PT4kZ1Cr=%HbMiuaj4bdDEaSCRK8N`OK40s#7MI3n z@4K5MZol2Jg6|i%|4F`oh|k`1^RB(yA91$R?4Z`mQu=?PJ5DN}`K|a`Hg@q{RBtQ% zvwb}`H}=!ym;HkMR;@j^l3#b|=}f`<;9aCU52URbllsk1V;^|4+XptE zk^1g(XZ;nHzW4`MW`|d%(_ztipRhkfp4j({^o}s&tGPAym{zl+^|#RKtVI4Q)js)) zr8%6rHAG~ni}cS@Cid$@TR+QVe2LmE{i`~3wjxd2tZunH0;1(T?Ij&iN?Si#=>} zbDkUYuD(LYwCV29mX7!B2^0FeB%8CfF$dj4zb`xNWC?jC&%{0SHMl44Q9k|Dt{Z4q z2Xej{8hLk_H|6_HaldG7_CLXzXNyjeSND&eY2J(a=k2czzwiRH%+m#PMDooRzV>1L`;hfBuXz ziT>vhJl(H-2%UF|&(!=W`E{n|Uhi8m*@F8C4|b+z!>djw$`-gYHJ>BD#ZQJG*kB;pw**Q19$A`J6An{z9#ND{MBWz*!RI zlTOL`7>gDr*4h69qLc4*zJb!uBVBbrth%W$HwL`>qNyX& z>C)BlxiZR@P_}Lgh#xw?sNzaV|@@%f;o><~?V}Q7HGHToR z{c~l|X8o|=81VYTlcUqhHT|92@Ak&a8eup2^yhV(U>3^vVMn*JmDqPxKGh|ER5GYN zVAcCteAiz>E0XDa-}Qs8{t?uFroQVES0>?=>Fnpb&L_{A_^$79v|fDW%zf8clsOyU z^&#?GpLxD(N0%Ej@SCxdZHvPl`M&FD$r|HUzK`}|FCu)_zUv_J7JXC2I7Wt%rJRnH zj?j6-Lh@(!02#>HiGH-r&qZ%9;MG=GWw zT6f?q^uRLp5spsk9}Khh-D2E{$%){RS@bWM^)4S$I4aWi?SFI*#M;AuXOgD5Lf`ov zd_Dg^qd}m?U01+Zw`qrR#ocopTl+rPt!~ zE7Io*%dW-iX@8z@&)+)bpJdZ z1MNxw^*H8Z(DlYr0lLu|yk{2+FWOTj{s#M|M!Ju+J?RwW{$}F-2j$~63F%d}^Fyq)#A9Ox z?^+Mj`7!->&d0TugN@SJX?=s^&*iV#Sa__QsT{>T9&{~#FkL}=>|7ONU+YKmJ;Hb$ zpY}*Dg}~9zKYZD%iM69Jh2X}oY~)|^~tf_ZB5^4-St7{Wx2Iy zWVZh|R%`A;PQh10Tg;vaD}1Q)!}R|(3d7&S@Tc3I12#Q_%=)rA*tb#5TJIJQ#bf4~ z_o-g|HhuGO1+;CB#J>LB@wYeBrS&Iebd!QSZ(|m&%AG-z<&6me&-$;+d2Er3*yg>>tG`f7#OSJo%Y*Q+M;zo+&1oBjUJ zEB5cCBcH;j7+|x!J}R{LVX{r%mFo;)f9frP+#$d!N*$b_AXs=f? zt8?_RPbS@OvZ65oKeaaB`MBmc(VWG`9DIv?F{MlQ{**kguJq^keR@FSUf-L_&fHF& z`k$`IMllAdZ=vEOn;P@*P%?5ganbq_dc*MQyKnDIR+Hw)rQEr~{!S{jEl(br^zDz% z{;;pm%sXdZH0JHwN@Q=1+70HgvhR*{$#Z<>D&f9`jCrCqUV@%5y+qsfUFBamc~H4D z_DA;8bcel9Joy5))7Fk8LsR(1le%<l@WqHV!>~R3G^SIUTUXAAbY+E?!>d`utSgL};U;UL#)@C<1Bo0o4mi&x#aTM3t zz?}l(AN?WyrE#cl5}!qfMDkO(e`Ein8~XZRWbw`L(SMQ?tx0c={VeK}{?j={%@=LV z!;=e^M*3gz>O_L{CH!hqi~sM9{`+`(;1Lh&TTD+dxCwQ z7V>Yk-jXL@U{;$|KX((@{lGRJX}j4gmu@~Cyza9h17#S02Avi0PrB}#V5>=1$KjTg``=(> zRco~R9V9Xe z((xUKYHv&@sWJ)a9otut4X;Xyf6@5{?FaFF`&j-z&FXnxJfnP)Iu28x_{;eU`zF~2lz2|`P}3bJPe8}|ma=7( zk!+^>e=`QAmgBe2x;L4aY5nKqJFHAFzVVxC65J1ycp#cv@&6c(W}bgkS({xy9{=bJ z?78~E)(`Y7hKKOW_MA(pk2v?AR@pDmFY&%ii<1TQ*8^G`aq^r-?(`#^{hY-UMlV^L z$-9a;tshDbWY0FtjpnzTqWgL5du8R%vmuO4v+0yiP$s9_ym+&d)Twqz|K{=(7`Gbt zpVoaj7_-Ebt=s`E76ABgpNZXg?KXDg6N`C-y0(b)BDu_ynsQ(NdayK4k{ zsy>$9eQ-|VAa*OVfi`~n`nmD*_?I3H*>SCj=^oJ7chWpwcYyZI{rbueR@T>r{~Nuf zvBftqZKd3Mo?t8qSJC;apI~k6hMCBMbV5P;1!PBMEw8oZ^!rN7xT`rNug)Po_c8a* zUd|up;*){)tSt|FKkt4k_L2M*E&R{SE!eltqUzJh^VwoL8~kUJS9`$L4(yiZZt?pi z?!96}!uVkGS@C4l8z#~p-_qWt_D=`c=Rx>?HhKT>Ea={O7J1Kx?&33`D;bp!C4WXT zdNzDNo4h(dayI$@%X!fW@tixSzr8r&$;A)S-vu#vG;-z7GR|x+vYIPDNc~!aFNmRI zp#S)PUZX)bn(r;&4SlU~G?nwsn$O6t%6E~y8&90V`YB)9o7W|h%)^P(o@b@Y z=ldta9?)7<)Q7R1)wq_=?(;2YKfz*a%G51v$|pYaUQNAYH22A^shy&~tZyivtD{5q z%H1c5o$$}_+Fmbhew;mCFh+OcU`Os$7##i{Sv6ykab|h!+@*d0kpG{nTbiS_@|(!3 z`D07{MSBwQx5Uij#_78z@tip;wD1)Z2OG#M|Mak%7mc-VKoeHh_+EbpcTq<3Ypru@ z-il1IR~F6Z@_nJg-;osB7g;Xd0)O+?!UnOa-A+z~S(t1-9-q4edzZc!gyip9pYA=Budul?aj*(((fr-B+3?C=hmU1Sypvu}^gp}p z=LffJ#8LM1t)DCV&N7|OM*cBpX%fuW^YV2J9KYuG!zam&jrsn(tv}6J@Y0>!pYoh? zq5Z3z%sX0x&W+%Q#$vLRaSktf|09}P57x#)@-tW)Pl>B(@IJhbdte9jp=*bY=@I^z z&gZ}J9y$6++`sg}*L9wQ=lb~Dd){Ru@AmQt>vF=PuRRduuOiQ##L2$ojSU~Wlc#U9 zcOToRJl++^dlNL1tY?VdAJX|OlaImuKw;8&lb$WQQ~n?L=rifXQ`P17>pQs9@|5+c z9pI6jnsK_kl5OP&m&Tm>f3W^izSGKZM_(Vx?wH)bfBB^2q?@Qi`as_c=)8VRPyJ)> z=#Hvl_l_J_l}7Knb5`kEpRyxOEOB$+54&f&HrjL5 zx|aIo4-X`)F&6 z`hO7U+(X(h@r-x%-3JM)@3a>i>F~ZiZQ1vaotwWGD_+L3tua{WT*WeI?VC=P5|ACvF$7Z#4jmF&=&o~w9?a=x!q=&Ve9Mn-#Vo0}5{ z-*H>CR;{xVwqK3^ARgSb(b|L+BCyqz5>aPI6sNu&9u(>3PfJ(elpQM~CiZ@lP?%hlLt_GxqH z&Z?cgeh1I~W9RCzePc)VM)o=7H5S&u>GYu27ygl5YH>Uq+~HzRl@}MQd$ma}H{W$pkV0pKLD{T$6~?956ly zr95YH4(g-i>wgYPa~iWH(AT*H>2#f!`fZ(q`WtB5oR#xPO}vY5vOlLf2lY1O*2W2K zmu=XGZjZ~I={czN)a9Rpx>q>RbMZN-bqd3SynQ~#%HNc$KlAmd@-1S2G0&$QL@#Ol z-b!BSCt;52=+ki{?$^G5pZ+&JhE0}UJPf{0`sDZLtSb0_+BE-9`}=cN|F532T6CY* zuJZReerL{E-R%4oY|rn{Svft0?)k}0vU`0#;^EKH#|3oZV7{bmyRBE*d<8rITF+S( zsB?EeY`zYS{lZxO%wGkA<~yyFi|58undb~+pLQz$Q?Ik_dA_P=!{w7di0;XKOMou# zKWBVhqvk5?Pzz;hktbEvSn|EPvb*6vd-do=sus5Z1I3@hg zK$7p!`>;f3&?zhbt{XoD44vGI-hw^31bY_ycbRyuiY?N21)77HEh21~!D?8*WLUr~ zpL34OXZ{|3kj;1#c*PK=Up5A4%8VJj_-Nn=YKLy=q_}FM{DEZ?55X9d&?fi9# z(do03dHav>pwRsF;;#>!zh0?0WI3L{K5pSNoWGWO-v#W_Sn$?XPuC~0ze&HG^Nk3X z?N`9ZR(zNYa?)Li&jo+<|D(b;;5z}zhLwrzc*^JT)c3u~Glq2QPx|do#OwOApO;>c zJ44eq-u*uNjm}HG`AUCz9&8NR`WC$F;(q|*JqX>&VX~zE7lPmNhi89NzGA+=tg+zx z%YR88-=EL-*Q0ZnR)@wgJbfSK?H(%X(D{1v88zk@KMR%S))~4Q=r6|5)x-_-pS|(i zLzj@{>{ z33DvJ@mk`keC;c&eTC1Zi0{(Nr~0NrXO<&5@pP$Y8~XR1t$%=Bp0bIHV!$kwNZFu@?vuB*&oU0hj zuh4hQvHm-qZ$4IhrG0s8Z7J!}=s&<>{bzM~{=Mvp^w^b@KPw;7`SmvioGEpF{VKHu zKPSz)ZGXT1Uee4Dq^?u>_1gPazCwQelhE+}`o{>X%|GCo=Y!&VJ#kLHZI?dWQkmet zLo4{dm}o6AvPYio^kg8`_0qL1pVOK+@&ooS@gDbyzCZdjd##BP-}^$I45UsoCxVu( z_ja>a%h^Zi&C~hc1J-{jEC0KU@_B9V=YOY2H@@R5e%5%>w-)`I_HW0mcYCO`CteT! z_sifN@)oZLPb1IS__vbl!Tj6kP7LNt)EE1=zx*ry9~3*UwngjN8lS#@o3E36ziohj z;^{BnmpN5FlN0cqF3)TAql%Hgaf~@%XTG|!oqJsq0pIPslo(!@eU#_q#L(dIuX47a9t(2{Nug4 z;=E6@&vKX_-w1*`!V#w#hDqt5a)gJmL;p*B4*waQ3Oc9Z`%dG$zpy@*3a(s@JP^*L zl0hr~CD_t5{-plbuZ90QYvp?@=}E2EOAq0P*t&M661~ryEgtPw|MpMRQ;MediBcp)_z=?^AYJ%?0T~1hqljwj9fQ7qT52bjK1CS+S}~v z#5Wz=xwy9N)s^qzTe_5eSE;ssfPOyl>0fX7)$(t*#=b%67n*L^N!_EqUYAW2)%>ux zr1oHM{fvX1D~ht6uMCr48Kk$23&O@C-pjJr3g^Q^_|CnI`PSU**yoUm#;daI%LP-r z#s!&?#Fv$2kB*$1&6Lf}ZbL3wk&A7}h3ZZgUQ&H_l}@lYWrjt|Docb zjy1u#ar~p_M*07?ICzx&UCWEISI22fT-u)EUK?j|~kbgLh0EpB#g3@Ue}K@@*a(loD^_%6K#;2oIDwSs5LK``tU` z!+pH(pPLOEkf(h-l|MOAJig)8IgUnylb^|Bf=tZ_{$G-x%NNcZkv&>7FPoV?FB{IB zm)({cbux2|^qvu{F~Wy~D$*(e54IRh{eJzs36naUR|;V((r$ zp|&_+pEkIv^Sq#&XT$3A0=`2E{$kR3!E-#n^vJov^F06g&U1q!JdY&K4c_6)?4&Wl zPllV1Njt^_&-0G_KAv7PGze!n`JL?g#>wv%@)+(%M@4-W%ju$`>=#JuTIA$miAztE z1T5tA#V1|-g(bn~{P-;+BYLw+f_vjS>RjB7B_?lO3yZRARA!;8qwAcg%v~kHNN7tL4zNkbn%R(JBT-aY#AQWU>w=_ zTTqm}S!EVDtaUD(@pQf7b8RS#$}y(i9n+dz7A#Vpn3sWI&ckGirT-0j8$ zdBR=po$}!h-ghxB<}*Ha@TC9Kbr;$AxH29e^WFG>PG&B$KgW#^@`d$`kGYJGIgF3` zdD){!Mr1SHq#xpUg!t}3#>b&~+4np8Ha==Evi9bUkGXDq(7(CyL0?wIN$jOuQvj5@e8^2ax82q!tEW2y^?OA+K^~dI6QGe79 zi~6IpH0qDlrBQ!;yfo^MPTD^z!fZ4Ne;Jzg{zy9b8t+xV?!7;~gz-`4`lG8PkMDPr zCu*CEcm325$4_?gu0QUGKs!S5CNVm5tpBJPulAgn3M%Dxp}pLpQtO!{Ot zG-{wR1)f*J?+W^23Oui*FDhiy%JC=1;7{tefcG0t7P4tuQa7Ajk=GX$L9V?^ii26; z&B;=-I45J0$BX>5HN`==7VMG@@`X+A9U9>(-kWI4G}^R^r}8{H!Q^aoEN9c4oIxkk zKzkPRbNRwWw5@@*Ev9XYkh24{c|Ym<_#GgA{~&UYh3&XN`ueGv>r_aUsj$eF3*xgR2H6mhWMO(b3zb+O0;}}o=IupTN@HEnUe9I8`K^;6ilFX)mjZF%sq63k??wTO% zNoFVEPipSac_NwJBH4K*nSDLw4j-w@cK6g}olbsy^8Ka3=5vwH4$2*Kw6?r02wy3| z7bos`Qo8Q(#M_4Pe;=!#I9D`|LF2ebBPLxG_JDr2aZKOZ7^WX> z{L*(ecIgKjx0~J?`JmvF|7m0Q>0!a)R87#0JU)y3J%^kVFD1V6ZhbZ?9 zWu8J-4k0VgQ06IQg>|)7t*dFB&eqiy6_yp&@BYB&bK;WVJ!&gs-uTp6794J<4!UXQ zvtWI2yyFe;)iE!UjK$ue$jr<G!+zuM7{mYu*w}xd?ux;L(g^wna3SoF9Z8 z@U{8MAe`y=cEyOm^!`@zP-oXx+FnokY?prBh~Pou56?iC&#cQnJKNLeFAWdAt~x5o zU*qaH_=X_dN*x_nFqd>R(9NA2;jw&)P08%%RL>^rX&}!Wm*=4o!M$<)DdOk4_4To1?lM+D6iBbuEXq1!zJUE}bKX1uJDED^UbnOzQ@u#)#h zj{Y0Z3D$t?ZsI%VV++*&jyDJ4;$-&ys&fH#E~L&y)VbKJ^Vt!>r>HY*CU1kQYYVn) zA$8Rgx75+!e2(Av z&nJmLyq0>`(f0eiy8n7)kgW7**Hd=Aqs<&F>?x-0$nAz?_K0Y#hsFl4{EtQk<74a{ zZ*bUID|x0k8jqKZ3jP!N)~3{A~4$KVM{zc-<=GU{n zQG+cfEu77h{NYTVv$5&ZliALhJf&mTTwu2S^H;fY)6ub%&8)>{H}P|M!d2Mzwb<~c zdD(CkwtYLceJknB{I(Onbr9R$JTLoQN85bg)fWVR?=a?UJG?U64v(tjpTi5Y?eM{D zd-tW0Z9l|Q>#yL;*>=`{4mZCc=%!DfrQe>T&z`4mj&xiW486qZZ1R7aIy);b4gN|` z`MAwfB`c4oAH`3e>11W>=-`i$6-#fj^!w9YuSZ9_^xH=VDf!tA@bozQIMN4e;Olf` zDn;ChM4q2Lo4O7kXOH$oZJ)3H*Q0~al&epxu%ku;zn?K$0gWc&Qhho_y8lvWgq6gX z#5!?|>BLJE-!X+T+NWzw&%DBaijdazN-Z)-`bd|KER9o{6mjJ)&dJkg;;qfS+D~SD zmY`3|>ax#|tjiuLuZzaf3!{U~>lsI{(&i7d?tf?qej>b?kaYJy7PW;NO54J2_F2X` z`VWn}KXDk< zfB0}aC%=CTe#`5FZg~7Gbx)~s`FVegyq(1VGf!Is-BQlFBmLN5?cSNFDz`O?&ypUd zT%6P2_bRTrQvBJOm{lI>&8HICPZHnt6tY(D($IIGP~Pr~qr9=cdy@ZEQ{E@Z+vL)& zD-S*rmp?uknc_Urs&re}#QQTgmUbrYDi7`^Pv#lQ9pWyE)WyNk?nL(JbmU+fJe|UC z8hkuLKTP2%9w#Z^lQ19ppKn2ay6J}_l<%SZNc2`2{ZWEG8%ZCO&Es9`a_1p4=OR1$ z-OT$X(VUF!u7G9}aXYwIQ{}f)eur27o8`f?t0o0Mgof#b`bmM| z8I3PeHY94($@1XH1x}6*GFD5V*-;Voxs#(;iq#$$w|cDUf?p6PTw%(^Zyp=GtbENA zqq5%|8_Cw*M7Af+Tkp~y9~(TcxV>uAMNxj_tt-ygOI9}%avTV0hG1y9H2)?bL9 zjhsy>X$vcPZ|{?{1LVoHL%RbxX}&Nxx(_)Uhn$roV$C&?(^|KGBMcYFy_Vz zeQ0Bae$!Y9>8EH;e_>?z!J|37jiD>X28S!*0W$V1GW8tt^E@(h1lbsWnd1}r@1u@R z`tGwl)3Q~gnO8NW@F~Ve{BnDyH<2ufdXs~bkR?%#5;dswH@@{};%B8ViIz5hS9v78ouk_NG zX1$AJpY(NcT(_N9*qLCDbW$AG80hI!f7=gHvt!5DCBoWwGiS(jkEpC8E~{bX`TpG0yvE|NodZ);2N+x$;>Yvr|oINJKN7p@zh>sFXsy`=SiEY-sI$X__?e_O<*l*JZn+oS&O=WwWteNi@G4V z{tXTr`F{*AJK@{&JUOSsooO=yokjcA^6%LBvd$x%1<+l^`_64MJH%O+$$Vq{L$$4m zvs`PVGg;wUc-q8I`N_+f+_3}7a^~Ps*LRUk8xuT6xI=kYLTiybq8QNny@pT~RM+1y(S*ch)N3%RAp8<&sF>a>R;&U zUr5~xPEkMesF}GsFAZvV=XxG^Tgt-PsK( z&N_Q%RXa=2WhJ$C_fq3W8Lugw3-3F_>3@&JxmM0&b0)fsGG(>d3*U?$jqj#PqDP_C z1+A;_udn*}xxodb=bd%#=sQoW|HqR8f{QdUi+4rO?({ zsrY|sbk=bjIP@Jo|D|sC|HOA-eCQ`pyRGfe+Xo-iKK~qXXF2*zdZ!{`_ZWRU=2dxZ zcK#{&^HDJM=Z}9jvA5&GgFE2M*GAT6zY49djH}IVr(N4BI5%1e{wcLt&Mxj8_3gUs zs3P9MZnBHq9Y+4Tk&$QX6WQnJ=jZ9~BQq1(V>1%jpH5F?U!0c6zAy!UvJ!u;B9YC& zJAI=YBJ1LxP;^!VToX4j$bIj_%~aZOk)UnP;{#muz9) zsrjbLnlFR=ch~D~q13iaGqg5Qwi$YxbpAE}e5vvuo>R-68^{oH`7(0((oFt$at8lv zG`$wwwcxG=H+91Y-FFnim*<<43uMer$tN=qJ7?y)!85ov=p#cnw!LSGn z3&FtoiRU;s@GN`N-RwahW?#EYd%)~-z0BU%OYDgqV-M}8?47;H{@V+^$!yOn%=u3w znX@OEH?vQ$cx3gxtY>Nev$tYob?=0c)#l%hv zGMT-Lz4mmHc{t}WS{jlEL(XdSa@IpYN4SN3&-9jw^hbKzyY88GGETpn z^bF}|8{Z1Osa+F;LbNPxa>Zb2JEyLR(lTH?3dXr$=mA63J(r%0F zy|oMZd{g4(2i^a2hs~U4Sdt7f#1AXx`}I!@+sIv5>nheHszc=!9m@WUbHmCTruZ+z zQLAsrG$-3M@5}6mwJ;A!Ut|BZ@)qf@md5L6 zarX4hEu>jk>2;QVMPGWNFP*uJ_}SWY{Yhc=c^X!vvZ9x%sLxixU#p8e7EZz+(Jpr3 zV#XKwboa7oRgq79VY)%*An1$C1m+{-_`$!hJjI&F`PJdL`mB#XTy;ye@Q0)cucdqV zqx3{y`UJ{L9?I*D=Xv3rGr0CKTf*%-%|j8H+7Le z)9vt!mhdYq{0a-d!ottf@IxyP|44`b5cHmL_@4s*rt_-9XQ1KZ-*j`tf6@%U(tZ4v z4*od(5b47IQ~~@4sEhoW0}j7v3BSU^udwhdEc`qTKeY1jb6&#ae?Rp0Is6^q?>)CV z+y@OGfA38Ze$ouT(tZ4v4*ocOKk35XQ2_s5>LPz;ufs1|!mqIKD=hp93qMc853M}> zB@X{?=%_3gF*LUF6Sfb@)X~_!Sm@ zg@s>X;pb`ip_PX}@E7zommePH!e%_%{~7 zzn;3tpIPtliX;a6Drc^ZCb z<>5c!@Gpg4gTucV{D;O=hYis1@gKS&!cUsvSGtei(!n36FC|_07ZevwUG7Iq($bz|)um zPp54&YO^gfYqJm2?uVHJKg=BXVb%eD&iwB9Qs#gS%mta>JV+R=9!MGQUyTP~% zj61-%9gJJS*bK%^VB84C4PaakM*N_k;|Cqb2RhPTlzkpQ@EskAUFmHvuTB>+M|O6lcV&4rr(RKB z-P>4RJ(aW!_8=o2UP0UhY>(#D=<#dyz2_Rv8a&0kxw+_I_$2fWK>tbRY4~(^X--L) zvA0umYOR&z=FcvV&8x}t-z}cG|IXaHKtEoPcK_6PU7%1}ye>f6QTS}@0`Sn*1zg;< zj`z8Bfn#o6;27hq&RZ8~Dv8zwi1+7Ewl1K&qC;6*7a(s~;?@Nw6t!v3Jkx`0?vL!N z#%^a#NiYgM$u}{qb0#GtkDH8e9>rvY^C!sL4dLt|_aI074=B%6NzmI3tqwQ8??xUD zMKY0jM(>f0@9B2y0(w^0_N@!FyK=|K+vCa|(Yq^m%&iOPY31Cy0Dk+?2}9UxWiPsX z2!7Qh=G9l{`z5RHjQkSP%r8;8XOlxqmtW=5Cy*|`r2Ogv^J?lLf2Qo}XkIE>NyfLr z@XqQkcHv^e#G79NE#D@GyRTx8rX+5q@_z+VIY zWBt|zY8-yj48PKS{FV;>IQ_U=7w9Q~zni+qpXqk^MN9Y<7Jh|=Ut!_rY51X)hku8| ze+YWdIQ&n6f78V3@EK_Me(k3BMf@kt@GITNZ|UHV(+`m@`FW}U{sYuS{>%Y~U$lf@ zVc}O;_!Sm@o`xSX;pb`ip_PZ9`KrnPZs_fD_;-MRG5B{u!^gka;U~@T zE8WL$>EMskcatvsI||_6N?qj7Y<2iWOZXKQeuaf!Vd3X#_@R}DzuDn$hTbNJeT@mo6hI){HP_$$D_4jMlG3WuLG!>@E7zommePQQB3Mmr)n_Gs_%)(Gq@zgB7Ib0RDy4MgGh}hhMaWUt!@_Sojqdex8OOT6y@_JNyfvH{ao( z3;r7L&xeMOzsBJw&G0MT$8YK2kJA^BF8p%~;IF4H@@MKDe$f(sg@s>X;a6Drc^ZCb z<>9~2;je+-Y=?g)_%~fp9nOY^kAKsx5&uau{7U!nTRQmT^cvEIe`W#v)2WO6nduI{ zXbHc-H5$GyF>T@mo6hFnp{&Hye_!m3; zq#1st`}i##{Bim?(uIFy0sJM@MgB~Q!!KIGudwhdEc^-!KTpFCtvviq4u4AgU|uEv z=M~Mx*jMPqm-g|uyL|=H48PKS{FV;>I6dX?zfxdb>^OCiKXaUQF75k?mhdYq{0a-d z!ottf@I%YT--GXI>tfuuBJ7&GWjG_6S$S1D zt(84A?S*Y)ZK;*@v=+`ewjVE_rE`ux4_b~dm}To?q=^TX?(rZ>XI(5#*E*Hed*i@$ zG1laizg25k+M82d@W8_Gz{2pr!tg-f+=~Y~cj@y$XBll>OyB17O}^34{vCNc;m_5d z)VkOmC-2o9xxl*EjlW`zsEl>7rb}2CyNdOmy4GJT?69+1dFLFOMjrHO=nO{Z4Avs1 zvnDZ(wTUULQB<;4QNfzU1lBIb(QmAAge8e=W?F4_I&BlS+&P8WtXZ(O5~j}C7S1FO zYa&Nk8{zJV;OJb!bMzxTKf%vilbO!CM7`EzMr_N}lU75%dh*rOqVEYWAiR+9f@n>q z&^d&cnp45C71ZCzn#pq3OqQ``vXnKG2Ffmm#v;~KR?52p1E#C|!M*@-s)Z`Ks>vn9qE*l6~yC(3k`Lxpmo|Id##R z2H_U1C;I0ET)M4ikZyDMc;CNJT0Fn+m-cDoCAaTS+EMDX`8_mke(&P0)x4az-28s6 zo8Pac&$w$dx9?v+B%0q7KVW{ZyrM%{o8OZ+Ou6~}++w$&#hGoSWazsN?cI` zeu#f9=+ySc(X&jadiJ5E@x765B+YcAbgr)(EnRyCE?v4xx)J>)o$SkmozW)0okOs5 z3!Yx1@2&1)bfffKv2^A2ZqMModAd>G-DHrZR@sLYj4|-*8^_*NW!2#dWXZ!Hu9{mN zom(i2?3>a({82jgjj~but~dOd#;EUo{QBln`7_JvBKsyBKH|VdD;C7Qw|lue6}+?x4CuU{V((tW-o1dm>%ra~T?)+x zXfI|DVHIosE2$Hk_aZj$1#DgqHt*T950Y~IW0yO+>)$IzKSMQ6T|_Qz8P*UtyEL+q*h_5Q zPWw)IxwCnUqh56FQO1XK>RqxCgz4*-@@$?e-IXOATS)4qDhUvw17wJ-X0r9=b4t zZj!wBtaLU{Pv7P(bmi8Px5<@TrFU0ut+RQ0`fJ(?Wb?QOv7LV2OFy=csMh@nc{;a! zUbX35(xh)J-P1RrrE5+RrRy$5=^Nd}(BEc+yQqi!nO#ojsxI1PVT~z;i(MFf6Ozw# zF0`u1r@pYUH@}Xy4Po64dfWNIzi=z~E5N_KAN~r5pESd-bRWN^gFjB+LAvm7Er5R$ zb&)@_$>A3*;a6Dr6&8Mlg`cP4hgKf`xeotE=xuQL*Moo6@ak{_H1cfTbrJtbGyF>T z@vl<4Y@SQsNV@Q^FMxj?b&)@_&fym=;a6Dr6&8Mlg`cP4hgKf`IT8N&yHfFA_Rr1< z$Y0d3}L=cz4Bo2_ozY+>3gJ000LY@R1`Entsq9JXBc zjx;+DF#=tUEJkzO3CwYkOY;H7pUl+A2l)LQw-f%X`}0%#02@y_AE0CO;66ap1qZ*1 z5AcwLoDIrs(j_yySc=dkb3 z<1ZY+UwCmkc6J(lHw8OenaE};kazhD^J}wqMqv*Bs|79fvz;rUKAmkp+>AX)6>YOQ z>;mlXLgFh^L2Rqd7B5TSFGgi;T+LSl zT+B)+WvzY{XyKfKRa= zp90(U;yT*8R&q*PSJ75{4V{zd-d;;Nur=3Gjyg9|=LYIrPo4Ks=Q`?KOPx*Bxr#bh zQs)ZlEa416iZcYgN$$|%yZRH{vvnNb;~2h255C6{e2;E?k3;w#&)^$8g)Mp#-E#mP zxA>guYjq}IpY%27?kAk%&gJu7eopn>I+x#0e(aHKO7Gqg)m6%m?J(bC3EwT3b55Wh zTRM_;urkgNjAT!@%$*@1JizyG>E?TouJLJS?EA&%`6PwXVxOc?TI`dMb`&h;lTerW zBrfh+`4YtCe3HG+C)tavO!a({Zg>8k_yInN@`?^+%_kvmSmNg4yYNYN)4#jvr;=#B zPV(2esW{jxpQMC-K43lxcVfcFC((D}<5R?Ge|=f%9(a92xSaF%4V=H<0j*x=TU5Ze z2@&rykJ1^JX$Qqx{mHGCe z(XH{3W^*8=`}V=oHIMby0A(LIS2w^uXx>bI-IZzfL9`h27KV3Lcd-i>6DFV82Wa{B zAv}&vq%T_0_nLo;e_GdUKhF09TL1I$A9vpmkY@Om?&G&~*|+%n0yhs&6_|(PzqOFR z6~8O8Z_pBcg@s?VgO%im82w{m^pAznKOy;SP64ev{rQx`Ujx0_4*yK>ZyH@4&hCeQ z(|aTNAt3VC213hhAYrqCgv|yLHXG>XNX*-O9z^p$o&VN7 zNp2mG`JSx=>^c5 zPhRE)Uuin;-`Cw5{|U~Jxkw8* zwRc2&Z{VKid%_Cs0Tl(kixM`!)_T}>)(To#D~RUR`ZhX};qAzuzWubP`H<1KpRMfa zS~<6t9*sq8+W(Qvb71)qH#xh z-ndh`#+^%NUpE@>w-@j|xl^_AcE;TzH}1#_&n%1#S>44hTr6HKa=z#7dE>6LJ{9cx zAEzE$9Y2rf*XiGnt=BrXCclZjHQk%c*h*b=vYq>rb$9YM?n$<>Mfm@Vv9Gc#+IIi#@4CEq{f!UX8*C}-6h<9d_40a_NIKBcKo(zY>{ST zOX>dDvUJUhTsrp=N9%rz297P=hd_S)-&Qm)f|kaX!Wvtmt+A!B#um?LUc^`=U)rrT zYi#}Ko>Pyl7mgPmTfH462YiQU3>g;THxDRd`x=2F$q z&-SjcBeFG_4%w{IZ6Wqc-=pZe6zzZ)VR6dzHan=}d>#wsdEU*2K1>f8)C;%&s(! zo7Il((zj{t4g41Ht3PknwKlf&{nxqJrm@&2ew@{*(tgln?z$}z?{!xNdcRD3XP?t# z={D%!>c-IpxpAbmHQ8?dpS{nvp#RlI^`mUdudywe5^$DbTXpZrNS-Tb2ewAzfHk6f zdPjP81({a9_-^Y+_ORs3sdpg;FS9PLIcGSvwHo|l?Sag1;YH}iaQYM|e z@Xb|}nb#*f_ZRYc?vU)HbPvk=xPR%6R%FKhXRUzjgwNOn%G zgU>{EzW!Qe2N~9#E0Iqk|0MD)wl&=r$&Q|dWXIEirh_Cq$eYQIo}TP{`(ru1lk0o` zd&z(OSz+19-#4Fd@N0~VuQDdSQqDL=pKQY~Zbi?uRMuwi!)H9ky7@71>^OsaZ+`lJ z*n9u*sIK$gcSeJdMpz*j%LEfymQ@NeCX`DlOexifF~*b|U} z$|F-9N;s6qnK~RwDOZ*$2PByi4yE>7m!qgE*X8(9TcOnDxLhP;>e&*q!zI+rNfq7C zckQ*MJ;Ml-)J{U1KjwL7?e**XuD#Z~-u15au8j;C(snLc!I;*5`Oi?Ekzu15gVd|@ zT3v@E!$M86qwD-v_X?gPU%S^4UZWr34W!@T=GFL??7{yK+j<Q%$&vbk(G*in1?C zxQ6g%!Zofu=tP$NAolaCb-D3(PlD|@a=00Jd<2f7OJC9KJA?)b&*wGJQN8g7X{RQmk&m(VlBWHKQa~-&A z!Mhb7KkTCKViWDcwt8h8{&?6`JFsnEABjI6cH$56@W%_`kB6Q88g}*`?Cjmx*&Xcc z`i4h;#mLCBvW;VF@9)Qc{yl3;|B(MTJ{8#|TRQf42m5=no@WQF?HxP3k;kjJZ+)7z zrLzw1$5z|T`nCs~?6a~l6E<^}|LmHLUrW40y!yYfrDr_HmOkjSmMzY+rDu!tZ0W>x zgT>g=sms{X?X#yWz5#u!&8}tI=sS~rw)C>$?plWDx2F6Zh{y{u(+hqgQ=`aHJuL)g+^ zp)73VW`7?WhcnCC#IbScyH3)%dazSS=Q7mF4s(~yIGb+)Y3JE|*j=1?HXpW>wWZ_} zHTmqBwlTC{Kk2@QjpIYvQaCgGJ$UiyT*-QSW|%l*OHsT}=W^p^OYxm4mM!SREVh&n zWJ}@9@CVpNWD8PR@MXe5#*4zaHk?bCbjFrKSrbX8cVWi-=s78ecOn10zm4)*EdJNP-@;uaEtKKM-(v3~ATSh97^ey^Dl6gJ1D}{3ag!Ui<;#h5v;t_;*tm>7CsczseGRg@s>X;a6Dr zp$0!?rQv7K(2U=mlvii**Mk2t{@`_#;m3d3?gbEM@GIVr-^7F8i{DAS@YiO+znQv7 z?`*dCRhIB8Ec^-!zrw-~HTWqj4L^IA27e9ZZLs*)fxiO$8z{q%zrx}t&fr(PAHRtQ zzZYLayzsBfg1?%&Nbgiz{3=WM6&8MlgW><%fMgHUcd^< z@Z+zydjZ55{EGMEH}T;2;;V=k{$*M4FQP8e>y9-e|5cXoD=hp93%|m`4>kBHD-FN) zQj!0StS`;ETI9dEqeM1E>ejiNblW-Si-^;?Bi&8XWNi~iv1_-~aK1?EMIX;cYwY5Fw_wnq;%ZBGWe`M0FdKUxahDNme^JlKJyT+mKkD=dAh`Rd=CDGL5rXIrBD3T+gQ+Qw4T=c$Vt8pfw^H1S9 zwwmq=!n5=v93y>97~OS?_AI(%TIUorX%8YfjdU|eH!aF~its$b3kc71(|h-I_IBmO zuT@d!3f3jdNV9~ri&(2vMR`VzWxq*$70*@-k0f6=y6;>0oio@|(0%jTZ(u)yJqewi zkk8gr^lg2MbU;VnV0H8j@E!K)=yi6#f#2 zv6%f=BY*TAJkOTQ{%0ZbhqhS49nHGi$}d0s@~8TJYaM@?yMp)3 z$Sj`yXk->=>s9v&{O1UMsU==`t&GZ^AG0GIb`SD}@;A=t_@i4&nE&Y+s8z ztMnaT^~H~OZz93BpqxC~YXo}S53;ZIBiNhR_gi=FJ4deF&E7=*?fUi;j=Ot zeO|uXJHfkCcwBwZ%ts}|;)~S(=zCh%nEp2!dVsX`@O6m1_-*&gXS)mixmNXg`=i;< z>u7&>emVQ9qINEtAJM+*m+XG5>-*}hZ@u$6P3T+mOSK;hwR4g3IgE)$jk!?qts5Kf z+#f^gkBp`MwmY1EsVlg>KTa6jqrdmpf1}aAMxz_ukB+2untS(#&ZwhXeETT6*$=q> z`(}RWY}R>(LFDT zco<60(wf%Qv)nrZ+&-$YXxE158?K%O_32rS=v}U!<&JGv&(c{x_xxuweSa-|+fbjL zCH<`lJ+v{zdINdhQp0%}`hS|9Rb$V~5a-&li1+JRCSGep-+3CdKFi_@uJs$~8yTDK z+R(0X6b>@CsqS3#EQND5*I8e1)^&b8OLiKwHVkP!_!<1+SQ{eeT5gO%Pei`>*S0t8 z+Lkz1PbA)t-^5E#^wz%E)`TB9b-SL3-Hh~-K?c9-LhhQd@GC6*iWh#Ut0(G=>)_f} zYq^g9eL$AHmCcs)oo*{j>I&|#mb*`~WFWuMis`ZKSWVf?d)?TV zw2v>p_&>#_bPV}%Z6y2DBiN_T3#RaQV?T{~;&PsQXUd2Cug_y2I?uH)?c@IodF~x6 ztO=bS;wyMID=#9u)I^<^YWQViLo(|-1WRx`_iZZ`J%C0P;_yMq>6nKWu}eYe@#q&Cy;g+7~9o3%;t+;2lMe&I?hn-qDCO^US1kli<)#C)px zpLt%kR_}du)P~}7Jh`)bkT?03i#Oc?{H$zJE0|BK_{Er0*HO+$*N>|CPU|%0RI^_5 z$sK2e&4(VL`;xNCt5VsdJb7hpQhD;fLWU!^wRUc)8He2AoSlE%)~uN6+N8!=xkJ2v z+?sgF9UG6#ckR+y&gmiBj*?z;y(@P}OCL62`lYE`<5uxT?$~pBb?N8yk|FjCcn{up zkko$nl7DfspRpwT=z=<9os*Of zX4aG=7#ql}f%E;Nd5(T68QG0)Z`K5pNP}$kpOus>G;%OT`j{(gjl3+Q{j2SoU$1o_o*B7EUL*HNt9i?uml=)RTSMD2KbXBN#<1G8qa+lMO77)hi?20u&+w3}V{A*` z+Q~D?y-CQtDOyW#_MbUt;0!(p5C>HU4@-p?EeOC7XV7jCJ$N z%%RP!f0|hTcz(JXUr%W)GQRY!ua@dhTVEUg$38vN?0vlrFXz!sYKKSqw+@f!yJco9 zmgZnzc5OZ%=?_q2#~XSF$0NQ7Ccn{wFC2GHW z>6}3a*pIFskQd*~w!V4wJu5Hnw7$U)@7YX$hBD^q64p24Sl^6c93vylx`wejFi(%< zIrb#&jd$1Cd3plB>QDXf8nHKKj;;T-?#^SY;BLm2tLF~PdqdgfeyZmV z_w?Lv=C9KE%bz?w_rF%8>A42yPxai7$M5xDgP!~TZ`@g4eE;3-dTy5U4e9a%ebuu^ z{Tn#nz_{WL|2(Z9T^aCk`Ck4ZLwEMQ+`u{^&G&K< z{b~X0r+KWOV(br9q~BTi{uK6KST|jx|Mjre>SnFwu+}o?X%?||;vG5je7=|7T5B2Q znY9+amrF>qB#OfXXDG3$F-MLjFX%f659)_1T;82bY@3i)eF|Lu%nbEv@o zaTflM*iJu~hX3Of>@=+Fvfc52wHn{c<6vus=Og4lguSpH-^&B|Uhc#9@&$Y^pNHq& z@VpbA>)^Q-46I$W_ww%X=ns8|;ngyHFH7*f9Eb1a7|>L`d(h<|7CnD-?6@zZ}b1W^}WRJ(%gGk%d@T4 z_wo(nd&xcc`OK-v>R7Pd^MP;Te5~$GcvkybthHEQzvaI}*^6IGyzCFs6}8`$`Ft(? z%KKhm);Q~XnKjP(Uh1r_&S|)3M#kDRlG%Oq`&+Dz-GYwo_q_~R-%Fm~X4_I)*|x}Q zY+IyFhO905HEdh2@t$9!ZEsaJCX#qt3wVo*wh<5C1;e4SOFyaqj*%ehvP8G!w7=Z|}Z-&M}z% z?=NQA|K=E+|$?tgne61wM4vYT)&`)G>KB@A`RY1*|9SA&;(PfveaVl%*7{x&XV!>{_qBi0 z#Dm|9e}{PCKc5Bv8`MR5=M5LXsf%_sVep%<@GD;Up$0!?rQy%B_+O{IHjBRn{Fm`v zY|FrZ+4?RLXYecDkKe?D-;00U;%~`<{}t*Yz4MC2ud;++Vc}O;_!Sm@sKHNJY4}4H z|4GU_Zt*vRzXJTnDZ}6X6&62n2EXF{_)R?cz4(*FtNoj^;6FrNq<0Qk{3=WM6&8Ml zgkBHD-FMV7lnT}egm%J8>;i}k%E&fr(PAHRtQzZYLkyxM<77W_-7i}cPC zi(h34zrw<=u<$D^{7{3RveNMD9$4XDM0pD={(0ap0sjKZ@Z&GB_=z+474OGy;=%95 zFCt#}=VifPL0zPGDlC4LCHx8tzrw<=u<%0-e#-LW?*Y4e7yDQmk1#5Du!;;(1h|C!loZPcMV_)5k`YX4^-zp87no3Lh>!8(C&tsG^qIXQ*@ReV=vMc6$Dw~YTw!V!l! zeV5EBBcAV)^?xrvqVJMn^G44darZX3omL;;NuTh4>x47kX8iHo-|x(sHl>w5PF|yr z>wI{~>f`?Jdo6Q)YtsxB( z+gPmibe7Hiw%5Jm)1Iub_>rCRJvI14u788MA2dylF12#B$jVWC!qViZ;w49IJm(5r zIqIG>@b_okJ$LlRQsgK;V8&;RvY1Ou*vL`BMvfA1M|-AyFK=KU`yhag3jOFk&MTGVxqkhZ`G1+SOYc~}{|reGv9Ew>1)h4J$cv}nWr@7J**$K@8jIS2FaZaFJsrxw+79e(=Xl0 zy({BoY!kbfa}2d}&aOxxA^%#F&-%+4eUdTc`PaV8Ii_8u*@3_wZSmBv-&v;p z+W%|!rP|W{ZqxP_Vx!v|M?_OcM@Cb>!g&3a)0L@TVVpGS4#!#O;WN;~(Z{>c$KPeo z;OYdveN@J`k4hNt<9IJ)_+h6{<`KqDKUMP{zX89K7&dsbXYlDJ-Gwe6lkOPtb@-hu zA-qUG!pj2u2FIwrwfKSAGZ@*Fd2fkRMcNhQt0LbDcTC@Df1smmSo}KU92}_+YS|0e z%3i=`WF7LRX9Kcs9c8Vd%xd-w&|kk(N8PpTIc#OmVKd{Fy@{R;;9eIc9d%YCH}>(Z zq8Ip9(evo*yV2KoqOaGXuh*ilZ$)3<9As|;eH|UMzlv{=N?%9E>@OL?w~9u%`uZ6D zkLJJbSCsyW?%Ll{z`ZF2ktAyXS0;^&ClB$B#6#%8^~4__U413~la;Bbq-PU;*4|5C z?lsJa%covF@84%->pt4J*mr)V&Fbo3r5FVG(Q ze7bz?u=tzWPY=;X&4EE4j-$hihm*9kboqBUGkTdHb94W0%6Z=ET3 z^5a&Q7xnA%yKTPLNc)P-cT)dtzSpcSFY4FjcS@Ic&&u3OYw~1WWy;|_c=sl#{k*&W zjV;2j3)k5D6KcHs5i;&iF!A!Q@!hW=S&Gf#HorJz?orZ9p1O8*dw+t$LD>-O{Rs-^ z5{4R^0%fJ?!nGE^WH);n2LHXZud&r@f4i?i`zJrp7#|tAZC|5mQN-+P5NB*U+I#S= zMUp07c10VH?qJr{AGm$jKOVJLsPwWwx$RHdAo56I)Bc2WZJ72?^4!?9C@Zb~H@op#hs&%5}EGj=(}`|+E2*%fVk7xDBT=bbG0&r=uao%1e! zQjepbP ze?1HS7V09s(_-XPo)g5q6A?`fFug=_2a|53``Na9&+0h%;r;a$+j+o(7XX3T~^?dH^cO~%6yKT(N zXPETn44qm3TAw;|rcPs=Fg%-hjd`A%F;7{3o{cVN^g8hjpR}8~w+_9o16lUqzq_*@ z^Rxfpt<6Rc?RUai*Qb1IJF_+`Lw`X}`L}+K+~^Ulqwankw#MoTYs~!riJo;`?AiU$ z$-wzmJ~)4R*7f5#d#7(sRIC~pTW@8v(Jy+(j;%YLBaanmYz^c$V`~Cqi}9szPTcGF zS@hEPuCX5IVLi~zdca{lARokmJ6A&7xl*n5Li##D`y$e>b;ruftkIfTn>L{*=^iZk zV{-T5xBcH%dJMg{={$azZKG!XXX2WS5Bi|LnC?Xk)6V7FN27ze_pjXP?iK0HW?tK! zF;1(kPS10{4qGbt>oPyEV zL>%dhU-Jd zw`5Amhpoe}(@QsPV$SmJ6KuJ`_s%rmq|J}^duPPC^CR&-o#5yV#cO`_zK6zr6=q(} za@PfR8Ku{Eq+I*3o!bKYcmLci@i zKwoApd!2Rbcb>-1Y0=uyb zFg|+gqU1pyU?b?>9A&K!t__?c{4kiha)a*~vnJ^1!A5}I{vCAf574#WN6+qk8yf+- za1Z)xH~OoC?uxG(`*ekor}Pa5*0KHed&S5P?oQ>~#U~5hZ!ly3t47opapEljFQk|96BrJ7M&Cp5G=4JhbV`&jO0mAC@1Cg z(-yK3d=J`h*ZLuQ_DpuLtr`@bki8FHv z=Yah3(2Zw}?k0 z|8?N6xJT}Nf7{~MxuX>p|1$8`-y`?FV>>YIuX9J7IWp}Z1Ajl?faRQzUmr;JFLwJs zai;$(-lq?^@$`Q$Ugw8ow)NjbDY4dnT;$tFXo|)Yu3pD{cH@ zhcNhOP~J3)e+u}kz&|Ymf0e~goWZYnKYkMrelLE8#Xltr{t47YdVO2n;8$6~udwhd zEc^-!Kh)r-tTg=CLJa;g$}6$>$AQ1)(~)EeW%%V!%NN}CC(htkydS@b2fr6zM!ed8 zTo(MJsf+Z^Xp3KE3BSU^udwhdEc{S|pR&^MW3v%{Z_iErN50qY{*>&a8F#$0x736# zD*aSvtM0}7Kg^wt=3Z*qqU_!qdb#$Rl5JjlV6$ay`6&4AJ}`tp4&v6 z?t;*Haqm7#t(RJAie{R7U3`7uXpKFyL7e)4iTCvZH(qn7?@Wi8L$loTtGzPHlMUOo zliD*I#tw~s!C5zBvr?EopnWxW4&@97JmlLsRC|W*cNBDH<0xGHh0V(fCixf{ou@l*9KNLydiHfp~xHXWMUb2jzD|!KNF0pX{X( z(Nx38XzE3D_7mvr-$Z9WUc%jIW$5b@`2N}?bowdi^wYpJ1D$?WH1#EXHU93uSEl|h zCz|?m){ZyPU%$I9kh-x3oxU2K9=-c2diOsrL#JPYZo3G5cmcf3Lm!Tz4_BZM&q5zY z_wPmbzcvMZcoO>Xgh0wE)7as+A<7-Hgx5q<(`gU2iP`@Hqx@*T3opsg15Mt&Wzvh2 zWy9j#JIPb0AK~2rer{efCf7w%H&`n;q0vp=x6r)rnmOp{y4TF?F+K0<{O?sqQ~y{M zg_mgR-DT+8OQP_Cp1uG*eI9yx3_ZO9J)JgrpSJ0xjdX8Q4{g>>+c~shlD7P(vb^}s z_rTmm8(pSN-l1*Yrj5?iR&UZ~Z_q}s(?)HyQ44MK8g29nZFG`0I!+rk(?&;VqeHY& zJs1ywbzdNL^Svl#!qep_WrFo>u$~9&n_ztdtgnN$4XiC-eGROyfb}F;kAt-ttVh6l z2(0yBJpk5yVC~7F+#Jfyq1+tG&7s^J%EkBSO?;2u!1w5Ne2?1jJ!--C=rw$gUcvY1 zB)&(-`R-eD4&~-hF5hLVNAEp=oZP4R{DH{C9M&AqYi@afH3ea<(VQb39m!YvtvOnE;CnPCuJ3^z4Taj1Hz?x+%D;i1%Ln$mZ-kBAh;@h7hSIyu zH{NU-`5!r?`RQ}F9p=mS)A?{bTN{};?^|~n?Pub=Z`~2sP5aB=s)x2Ud&2hFQ#?!Z zOg1aCC;XvZ?|sO4oa^5c&U4p$Jom3pjQvPyRStRGJz>|U)7;%PDJPMh_(H>}#v zBxg{jZ@x{|u5$4cXYecDkKe?D-;1AR@lVTwe-d?(-kD_at1RJHSojqdeuae}YVcE5 z8vgSZ{{+e_v-nHEfBBI}vWznP_%A=>;wR4FSG*s;i3h(IKY@6)e@PbnW2lSt&KQed zWeLB+!mqIKD=hp_gP*d}@V{yCkEXnl7XJwFSAc&cW%%(|Sp38p{EGMEH}T;2;ztuN z{3Ej94^bECosh+^vV>n@;a6Dr6&8M|!B1If_}{Sj`$K$Rk)Qg{hau*Ol1TD*@mE>=#2NgG_v1J5;P>L+CSLg8 z%!2=Q>LR`Ky2Y=ugkNFdS6KKJ7JjI~Pg!aBTP*%I%4@OsUju(jaU|J78GigNE8X@d z&fr(PAHRtQzZc&|yzsx41^-Faw=&|KwD?t)@GC6*3Jbr&!Vfk0DJu>CYZm`;%4@dx zkAS}f{LPf%$6sRc6KC)%-jCnJgWrokPQ36R$%4P0x=8QTTl^|Z_!Sm@g@s>X;fEUh zl$D166^s7>ZYr%hcTqIdX8GihiSGf3zGx!zn$8X}n z@5S#VUifRX;NMJLq<1!3{3=WM6&8Mlg;FZ}DW;IF1G(mT}_zseGRg@s>X;a6Drp$0!?rQvV3_^T*y zg~h)N{Phn-k}D{~kH7xQZu=8w@GIVr-^7F8i?1SH_?KnDzlgd>?<}(TRhIB8Ec^-! zzrw-~HTWqj4gV2~e*xvqv-o4+?;jgU&Z7)J{{H1Ie&P&%#ryG_c<_7i3y2r~SQh-V zsEhQ@EQ?=d3BSU^udwhdEc{S|pR&^MAF}voP~J3)e+u}kz(0*L{P?RZe&P&%#ryG_ zc<_7iGl&=dDOvDOpf1uo6D)p}CHx8tzrw<=u<%0-e#%P2UvKf3QC^9~KMwpYpNS+( zD8rAx<=5QyC(htkydS@b2fr6zM!fKk%YuJ2b&=i~ZSkut;a6Dr6&8Mlg&%70QdEDz@+HnPXB56BlpO=44HZQ)%;q%w@ zf1BZ)$#2Bu>t|_q`Z;m^S>XX~Sm&fm7<|E04N1LDelgS`r={ z^X-`u(&|@*z4YN*4SfIa3Ciu;`XsyrZQJC4C%>^TloRh54UUpP;)==`9Zfwu%q`!% zm&PYjNq9}tF77=`obL-p6GvmkuO5x9d^H?+_1k)9)q%j%!Zcj=R-WhQ21Yf4oiUTp zb2Ba}!#SioVm^z%h5E}^pz25B?P1PYeAV!%ai;Y5^XmZ5g}3xBtZjjIRT)kRm}o=q z&gDklrFd@S-HC6n43(qy?c4fPV}C5i@cx?%zrT-nn%)m=A5uMf|7&UUZ12BsYp}78 zKA`unHc65`kQ+_q(Y~fG@bzu2Z1nozl8m|r^5q2n%xe?x-5nJV)UEc-4+b_SkJLZJ zyZ(8lE8yqwekUp}H?UFddNdH2ZtCQD6K$mb`la>Rm^%gnfxy=T-+CC^u-li@eKsby zJBRY)&KvZ}_vi~<^zF;oEMEyG-hCU}q$aHmq#ipLNK9N6NKM=b&XtrEDu{Qy9!Sj@#(i3^(^hXpQ{k(DRQ^@|ziHnm z&#t}6^Yz&KgOqQ>B!Dh5e||sLE6ndt>0mhvVaR z1-|jErvksAb`w7C`i_UoNncLBvDmi8Vt1-!%(nUPwpqNq_wG9Iwu9#!c-wh?4!pa- zy92ztz`MhTH#h3yEvBBXPsatnlCLy}-<(uKBT@<2oEuMXrP!*KCA_VIx0OEL##-JOLrw_Z z_Q2aNc)R>Sq+mO|?Gj#ii(B3JB z zo)^ROLU>*b&kKD#ms*|~V@})Xc;RYzUIowfqay{&;dvE&tcB;bmgo8c2^5B$%E|GDr#AO7e1_#f}`-#eQ9`7-!l4F3z^zyJP7!Cd%X z2v1Alf2rla|BI1_#s8-=_;=vnA?i?4m7p&=!@n#cbN5lD3#V z*cOkLyYofo$ja0gau~mP89ZKx$LsLe4#svcUV>hNc0xO$z0h9hb?9|yAG8k|q`n|^ z6+#Q4qoAXp#n57C>9ACODLVIoyXVvJdceo)WO$9i>vnkE2Cv)UbsM}!sAD?3MyO}H zkJrhT*AbPeC*k#xoWKl~>AZm+XZH5c>9mhKGAHqR(0%?$c>vuv*fen=>lbvU|4Ba0 z%>mlE;BVaj;gzmVr+s&$S9PNMb+Rr(N7LT6!M!h<`mf~ec-{>U=wRb%-|@6>G4ie$ zc_$rgx6QLVn%b{CJ2Uc3CeLK@j7QdtN7j^+x17Avh3ee2rp~%(YM1iYX5^nv{^{hO zPX6iSk3b{Pxs)@Pa$4zsTL;TAWo?b7>QvU|jIw4^)@;gJNEr($V=;6wbR~2pbQNW; zqRa-$tVu7^lv@)`{g%qz;4627Y0m`b`V*!-PgBQf>R3;i>nU?1bR%>dbQ^R#bUSnp zb?u=p&G+kWuglc8&Yb!9Mn|=;zG_?FE4IEf)OUvZ_EPR%%H0p$4?PS$3~huqLQg?Y zK@-%Qpx*7&Ta~?DQ+HJ~HAB2E^VPk~)?IJwK1bc>sQV0MpP}rt(6i7B&OgOT;U*X~UoIIMgTb*Rnko{G1wI@LERLNP#|3a)07oC?_fdY3aUEn_7eWi6 zqoAXp#n57CDYO(i89Es{6FVR3*8Id58V$v3_T2Ogf>D?K~F&w&;;}h^bGVYIM0H! z5}Z@+4yVC7C7OCkSSR?fPB2&_e6P|SQ@vpA1?vUsxBJbm+nuli|!bW=(yQS(sTJzPJUmNb~ zi5;}1!Z}+??mvVbv;jMu8OL5frVl?m%+<59rQNSSG0FD%Nzp`pXy~+Om3D$nI|18l zaOkvjjBj+5*|cTIhk%RM->ye*l+2t(yO{S6A1}d&#{;&vZ*-K{{3V=^;=SO*m3S(5 zuugyaHA+9mrXOR|j}9cB_3`6R|6i1Tq)k6Ens|ir>CcI;`FQfDKcn<{Hhmuakk-7v z5&Wkf^MADDp#49Nw)FZvd@BCv4X@u0+Q+ZXquSPx5APZ{wrO{iM?7Yo9~y*BXd_n*X=S zzag;yAZcrUfO0=6K7U%SJIDPX?XY9NxA*eV{qvXp%AJ$a#z@=X-p-56kJ_)(9^Z@X zt*HNZGB0YMZxEAjkMAS$e)+3zeO?*E-|+gbuZ)=X`i8N$7L32hT%oyJb43?(pZ2Db zV9=bQxk`I$nkOW4Te?bSzJx65VqWZ&Zcr77|F!yXz?H53dFtr-l9?lrT}pGEG|bOU z8be+DPm|@OGcxYHC*z)AAI#2ap+Ox(`eo-B+Nqwo8JyA?gfltD>ObI2)<+{A&L(hb z{??s$(odS0C8p`$dSqx=avink9Lgx8mDtVDNNc?VRbgn%A%58dxU!VGs^ZH0he0v73zo>QQNAUUt zX*^zkf&cdeug{glpUmL(LH1idiq}U-aDI*qkl_GHnMnKK&bf7(~~`my@o zi;TIx57|RU7TEn&GwxWgyp`X`n#|Le>C4(<)!``}tL$S1?-!sz*rkVa#%Z>vw< zWVDBLms(SIx}4Vbz8<^T)-$#Dnxj&H1M0P)*^A3tvSaqp9Z{IylPm&z7 zx>_*T$$f#uScBi+cJ6z0$OG2=0|k(>m7OZ4-Ldq)=*=LHEjBr z%1Em+HKzqAx2k>!{k`pn8*>+*xbv7B4>lv;!0^M2xnJZwO8&6Gb#FbZan{JYDu9>A zf`L?iIGUIk4kU&ekLp)ZR-m|}N8vPJVKi2W$=le%EE;T;F^e5&f)1NrMmKYeT8SN$6a{IjT;~UTG z{?CthJYS7{vg7&5?=znBKl$GV@<7H!|McPnv9b`ryYaTmELwon=cIefwXcD@^G_w%ixE-f4Aa zPgjsWHBq|45IXza>Iko9j=hgbrc4F5Po{jm^zR2{%CkQLnew}td}Z{@pURX^aE;rAQJc<))|(Z`i0Hnh&9u`txSM{&|s2lZ6ndbTcojr2*S9av9c!~djkoIb-E zf%RdUt;E;%#)m^|UE3Z*`fl7IkFg2y{s(#RtwA!{a)4)KY2}}0!}i1bc|3h1yPrqb z0e79`*+lN-=kY%%BhAkvV_Y?~J^ZTtJhaZor$BpW#&#mUw3jSfgnzwXR_V@F>T6p2 zX?@UL7KnFaD{v^M8#{q)X5HA_9PB>b*o&RfQFBJ0{Z{i{2K;Ay?;z7AC3}`>e{iI& z0b{N`z_XnRgEwCFUv&&MMh4y$XKmX5>hAXx2K&L#V``vm+T-f6Ed9XZ4#@A~#h)4Q zyZG*+|IR%EI8-d!#ytrober|^RD*XXDMxRrRwQOh;zT#`rneDMeFZiGknILp#+Zu z#)0!{S98pnt;ZXJaf2@xeh+~!2);hfF9s>QkFzMH;421SDfo(g_~ch1eB~;abE}-K zzK`=_JuRHu<;>bJog1^~whXqGC6N&4*gR~11h(f^WMJC@rY*ErFSHk$f~MxA9^Y;; zhQU||##DfFKwwPK#y7z@7K}w;91F%GA4d652;+D##=ywgz55biECJ&QFmkp(+zP*~ z*7u_XjGX)TFm6=)eJKN@cfQX%ix%cAQkb(y_AJ`tt1Q++uvUZhI#_$bdL692U>yb4 zLa>ekYoQM-z7hs&DOeBPA1`bK>tV3gat`<)SPz5s7+8;4thHZ^gbog3eG;sJ#Ti(4 z(SEyV$09Hmfzdk`H=gi#!kmi>b1v@jN{e|Em}6jez%%M` z3bk)Bm_uOR3+6bOFXIQa3(TCy5AO%_evA1sen6bn_u6-?+IK+)<~W$+V4e=P>0tEE z)y*Y5m#}xPZYklVU@x`Ui@;t6_D-;00(&ReFM&M>_P*gzu=hPVIKJdBA?#zpUX35q zcCc>)dj;4xfqfg;cYu9|#a;n+&fk03ySLJR=VoBv5BB|F_s;yS1>;&UduRT(5Z*$V zGk;;u{CQ{c9uHa`z|tLphl}uV0Uj>G!v%Oq4X2;OL+VMWj|cfhhzEFg+VBxt4-aeM zp&lQhRq()>`S3<~*l2mE$46+@AP)(6c%~wQhX#0PfQLO`+yln_(EZSZ(1XxMXe0C( z^cXY&O+ZgWPxC%H1Fk%wPjq+X-tMP$9bS6jENLT^Itx$(!*wTz5pY<9Q7(^=ln8Q#xX-p?7{j{{Fv;mLugt3ICO+o1P@EO6>Z z#tY}dQzbm{t&oBUJXONee0Z8~dE(n3p~xUli}il~w~Y650-jF5lXv#fJIn5!edH{A zn6vCU`xxfzV_4_ybk>eD=sJHMu7v8`{aonWCsL0$Sl*D4-G|_<72Zz6TPwVshPO_5 zy995Y@OH_^oBS5U8#3AH`c%AdD!fgGH@+iMFdp6}!`pOt<7~Gp|34oIjW@g*AHat+ zekNt`mVmbeyx}JzACa*9WaJ}*Pmuh9!t#BR-_v^NdgvDD7U(wUHs~(sF6bWU9_W7P zet6t&c`S#=I(SUL<0*Jdz~d=+yapb6+{=xOL#=vnAF=sD;` z=tbxy=q0EFb)db_UU=oqwdt$q4c*o7dIDaL!RrZlJqEAs@Olnj+u`+`kJrhT*C4!x z#>5M!qq|JyzPZcz!N563OWi}1TBJ=LQA3Jq2r-bp;MvLq0^z2&`RiB=v?Sx=;FDl$5&ZCr^06p zJ{#flFnl(`=VACf3!i7;^DKOx@$os;<+HaLKCAKjD}~Qu_^g0W&Z_1VgMB=Fj<m#fw>{(Lj??zwlgzrxHo=iQHsUrf7Kxac|L+3;1Ll;69LYG6ALsvpqLf1mq zLf1prL$^S;K(|4+L3crSLH9uSKo3F>LK}vs0u9h@tQ}^2-?u033Z&k|r^u{P*1KyHXGy^D>y**HtLYD` z=?|^J_#czsDRb8|@)a5IdvQqPkftFRf5D$-25C0YW}9fU>(J}ao20!-+U>#k*ZgT` zk#-wtwvi^toEv1$&4uPdN8xWUDwYbY3dXnk^Hh*$2YGgoXDn&QlC~IH3@wM2Lno7W zGI=Y5@qh2n8zb)?^6nvTgtQUT&W6s0&WFy2E`%Kk23aB#$L); zMOyiEtcI?JZiH@xZh~%tZijA%?x3t4lobrdpYoTrfU*u!)l?vojE%<}Y&*Wge!?!<2cJv}Z|s4tfrH5qc4N33>_YKpki= zv=^F!rl5V4-ACE2f%s>AW#8bu+kpRX82KJXz869Zp<|(Ap~cW*XgRbTIvF|{8i7Wj zv!S!0^YP7?{{;Of5D)q4=$HWhQ{X=Z{>7wSOxmT;rO;K-RnXPY)zFR5jnGZdP0;Po z?a&?29nd&54&6&#d#P(X=ViX@>(7&@>oj$prmhCkHjwr(^f2@U^aS)2^c1ue+6p}b zJp*lrwnHyKFF-q?ozSb$tI+Gzd7V0$k1zS^EC@ss-75l#0iUWA=~JZdgZ4qg$j319 zu@G7a9Sa=`Eru3D%c14a$FPP1KkhZ4?PGy2yKKmLXSa@K@;Fe zfP?w#KqiiAa9jY#1#q;IzLoT6pl6`%(01qr=mlsev=e$2dKG#ddL4QbdJ`H%76y@n zxzJqbDCj6?5wr+e3N3|>AD+q||0J>@5Z|1MXAO8Rf#(u8C@dLo1<`(7DjL z(8bWj(529&&{fb?(ACh@(2dZI&`r=y(CyIe&>hen&^R;>-3#3d-H*KA|77YbV7fj5 z|FcQlDZB!IqAL7~s-uZ(YodwH8Q6Gcao)6oyNF}hz~*r`@dA947BN>XK}PF5>N1^= z$M-0P?@=Akc52?lcDN#tS`rMT7Uu?1UkC?Mi^c{LpD)HAsWg!I)$xJEf+Bp8#s*SJ zeaEJXvt;-q^;EfXzo0jo2s59ISd1Uf!C?t=U&CVBXfADa@Co|zutffuCkB0y)?hCr zobUHV@}Bv9k-TSPeZELM`x^UB6Ddi0f`;Y~N z=mUk7DcL&Z3#@PYY-z!s2d3l!=1%5K4^vh3!t+FzzU3e4r0@qBmYa=jhsp{T$s0AIQ`- zWKjXCatEdRg}4!=c}-rV~2PQJH(`^mC(x6 zZy@s?n;uQg2}UDjlij;=qzB~we|$}Oq+PpGrbGSVv&ew6)O(3~FCoL2JCoCF-K4oT zjX3Hq&y`F?A74rO<*Y?k@_adKk#g)E*ggu%SzBQ1z;BDW15CNBMX0;Cj(YHOEZo9f zKAQsAO7bEF>$%Hk6ZyB1cALG+r)6Fwv_2|3!q;WTfAc@E9t&q+Z6|#@`L05*LLD$W zU}j$P{BzZYx;Nio4r>h&iWe*;{bDdL<@sVTmx39)MnNfNIs zsmhNOY!7hf3i;#I8F&4$I;) z>^oQc7J{`6tP4pyAFK;WKOd~vUcyCS9Shc?L9FJUA=m! znEsQTU~x}~rkLEkiGcj4!8TYCRo z^43E`%q1h)i)Ai}(Y960B`cUqmNAzcVJ=z3Ttff5wt~L440&*bxnz;{V((%uX=5&_ z2&DcG%_SA!jPbtD@-ENvf1ca-_3YX_o=e{Fe!IaTS*&+nu%73<_X74(!mH^Y+DBPE zC~qb)mq?x$lF#9Oo5Jk@_GSa|`?et;E0B+ykm=hfV+ZB!urj^kDfW*BW%`#lVN<-{ z%1W^4d+_0Q(zlZj8Ia#gzmwdUWyh}iocIvlT=2%gyPEW?z`L5~tH8^ikM>9k*yGVY zi8uc|%KQ@oZ|81g&v*LwJ>%K-ym~KE&PB?&3B3tTQGbg1S+gZ)_}Xg*ajHK|{bgC^ zzva}=o=w4W>Sw>ktACO=m+XX3?lddh%Ulwttr)WfyO>Mjl)Ilg_e=hO^YT-Xt1r<$ z-kj&*>;zvYX?^l7jC>o=+a)Wfxp>WZoaSq@LNJHGyp%E)(`HL4XEANI7|aX7yqIzp z4q_gdQ|h9r$+X$Gym`&T;!uu58O)uUH@*7~HUFDCbkGIz(F44>*x;Gs!!rszU0LS1 z`Q)Dqp81qH*N116eTTKt)GvZ(V2%S%+8p;`An~V~<9MDn$I-uDlRV(L=D6>_=Kdd; z<0b^$K6r-raE5nJfA-JKe?*-fWrKae^w)E=!8zLCI(e>>NByaYov5u2Rlb%0Wjo_ZFcqn!6mQL#I<-g!0&f^Ze7ydm9l={Ri+WgK&3?Crgy~-}}Jb9<;^4{7Kx16ZFHC z%%6>5IsvA`(3Q+>JD5Lr_~uXYDV#Qcde8jxr}xZ1fAY-CpOh(H^gh_rP9P@+=g$V_ z&#G`F&=5@p4>Ny)A=K#3pYNc1Dvx{j5%N9(=F^msfR{?zY&Lzhl6ITz_EpWTP95`S z4ZLimUu*<>kha*2jECpL^gH#<l}Tua&c|CD8}Z5#UhTXKwP|48SAXd*yg9O}N~KX9_2gDd?5omb<1 zjpR2Xz)yW4(B$3y>%Z?fzdPd`TuiuJpKRv$2zQYr-=LmMzd}uy))?Wb| zzu(Fqwd;>4pZkg*zNPW3`-ycwvR7Z`UBisY_;;}%=Ldw5J_gP-|3|+b@||BALJxVg zU3)#cGp*w|dPp;R$Px6AdECWYj~;RyU7;C0^AreR#DF7T!_RbEq(0co98-IiaVvFj8<1teP9FoHu6%Y8OUA=LWGB zg7uM~%fPxGtn0yg26_hC4(4_+uLASV!TG@8-f3}%!5stlDR7^losn5)J+KzstHHe% z+^c=KrN0PwA-Humg}MDc=6Q2Ah50{cJoPf4;vT>p z$@%Of=XWv(*ABM1@Ef^aYx}1P_Pdp#dHa!vph9!E$(53J>hL>0Fo59qHqb zO7EfVgY>mO>_zW+B%@9DgJC}y3b92OV&@$T9Sbdn7DLOS<Eg&3vY3F+Y4`Tc+(!}TzJzy=-ffx9v$nxgU)yGmF7-_zyleW z8^PQN=G9{3#=}cByljJ)E%34pUbeu?RLYzTFHyrH&9jNQR(8F8 z!NgMCk;?N&iUa84*!8Ak*XzoQpT!2C{u7|zw7%;8m#tQ7ey2L%&Ueds|H~^=`OKZk zHSW9VSQAaHMBd0QAzoC5nHyKpeyeD|R`RrxM|0x}n`cEd^#$cwmXT*IdDfEWEc7h& z9C^=?S98=7H?4W^OQNYSl3#V1wa-4*Eb!o+;S66S|3&g&f?k3;l;cp2=8Z*z<(RS- zMN`kgd&fKV**6n4R@@FA${OwU?+v6a09Hfp2 zWko1!Hgq<0K9qgHf%{*V6JAbuC3UT&F3HGg+3Pa(O^c?uSVVWgrN8sT)OVQr)>7VD z%3BXz58VRY0^J7P2HgeS1>FPP1C3K}oO&hSCfuRk3DMNw(!U4Zdxhom81){b-UiBR zpuEG-!_X7Z6VOx8Q_xmuEA$NX4D>AYEc6`ppQC=soU)rn8&4uPd!_Y9a5LyTwg}fMrycqWpIL1X&2f)!W z+D+$Fv%g)z-J6qGQ&(~K<_vfL`^rAn-piuc4Us8Runmr8jBEY9f_?9k$edTAsb6JH z(arh*8d;Fb{&pDI(3U%B&t{yVV<&WHau3f(%w;XRBPXFXt!yu{1u$PFknxH9V{-=W z+2m6=Z9VQi^RLIfXa4m#&&+z9GS%+pol_QbW5TS7w7)%_yESV+9SKb5Zp}z#DhCXq z*_Fng{o!x9GR!F(6%Q{CrUDBGZC3^5wDox8XgTdY-j$=WBRQer@$R+&eIk&O4b-g1 zUuS&+&#kYpHUU%eb!4hfo|-;^?Aaol5%GadT8}?^SL^Z3s$Nj?1|+&*R2u;kU9Lw}iT?@A@V zxrb=W+FMQ;>(9`*Te96XhE_m_UVnB|kG_Y~lZP+YA>GMUe5idQ;N{Mn2Rg|2?VkR9otD?U2VLF_e`L< zpbqqhkGt$2W=ve3lI8wIJFboIC>|FhIx$XbF;Q{;~i@2{XH-5Axhi`Qwdy&7L zlD}oxxzIP>WnS)Oj=hE)?m-vmX3lkxk?*$z;-6{_-YTpN-ny?faBDI;%gm#E71Dnt>4B*se8f#?4KtOp*JH>WuI4`4rFXFh+Q8n zW}kJWoiB(pKDVKqcy|oGTSK=vdwVM%KDD3NK{@~Me|>+Sd~kK%Wg<9q--GUF*II=0 zbP1(7PnwQt(NsO(?=bpG8mp5U5E|?B~Q)!3kls%Pp zsU$p`a3$f{Zo6nN*9isV-DSk(Qf6*6^*3FX=P-D}mS=mPx7u(LbEEj;F1iP}o7sKO zGqmR9Jp8HteWg4XUk>4LkTPg9qtncwe+*+DSG!TC){e;)?70yCs4dfLuV1BY{u^_> z2~*Bi?JMNjKAA@wEux)yzM1+u7Rm1YCHm6R#(wm=Z}F|h4)l^?^tH5m`N;3doTGNn zLjK%o=K|WK54hiJ^nAu;x0j!Kr7%Wc{Pr|xH0XW{??vwD3;c0Wyg_&M{X%1mtNlI9*kf0^3qmLh`fc2W9px{koGz9m(sTIE4gp*2E*Ov zKD@esJ4UUnRlNF2rf%i>pAUZbufI*Y@242gq4-hcn`DmmQakB4SCG41thL@9jr=V` z{z{h|2Ts;B8i()K2I7B;E_$r)p<6G99=g?BH}+OjZ0xHJ{im_+%eNfHol{F6E27^P zNq&CW%HLqTAea8kyDb1?nEo{|-izUpF+Y*Gqjk$~(f1-<^vljkeE%eg&B*GdlBv?i z$9wJ1n3imJXd6@KDC*>WJjULU+5($J5~UAnymr#=SH{rx6KH$h{Xn~G%uZl#pda*d&+jqX{$<*}nYQny&AB%q zp>@Gg+FtwPN!rXE+we>u>t4fqn?k>+4E(d^N$LNA%IH5oa1t3syGceh{~R)kw3>&S z4h7l^m|w%puLZ4@&`M|ylzEdm6WgylPcpAcUO4m{Z;hmSWmo8+UHY~zWDd#F7T=mU zs0+l@7WZ*SGHp?3+k*1$wjX{f97q-BvS&lT(K@FoR{W~IE1I9ndx6iRp|9LhKTB34 z*Wgw2s$@+=Y~`yj@!iqSATuPV&D^NA)7-cSnMwMMnj2}$zo(r{JJNp|W5M<&+F$)9 zpMD^I#kJvK+^GMwk@~v$?TiKEzr(X_?(FZpeeQhLH+O!8^JUrQ%`+!|xOwww%2PXeZS@50a?&?%9(zCgyxB;7 zL(QANKtJ?*66{zrz&&lvUY zEs^Y?J~MagF5?J!kab^7o3~@`y)BZ(!k9TPX)J1=JinB41iE`EX6LgEoSmwRv@ez+ zn+!JHPvhgu)}!*WmEq|O19$AwZf4y?`fTNRd}-Vdh09+icd#2-Dt(>wu5HNhmfzDc zrv9O9bqv)eX1@5s>;V~>cGf*KfwS&;apwT-{t9h!k~!lzbH*~x1sq|%eueq^B=hxg z|J_bph;bg19Emt$b>S$iKZSb6h;#YEj^kH3xm+Dpvc7ttjyf%PJ~ z#C2?qz1SMBV{7cSdp6YlsLmQvx7HG&?{RL$iX7HyB$#Z}beM|PK%-t&3eRJ5a z3bPlL&t8;SS8%t=AO0aa`R6ilpP-$!Z&{SXZ%%5+eV8Elcc$Yv@^q)KFQDt0^B!m5 zp%os^z(XrMT!e=U@Nf|xE)4Qu_U_>!$op%&%DL0>3~ULoCBQZvY}3J339W?Ah0cX8 zhAxIKrHz-;#@OnTb+=<2XzM!gsI7~@8KbRFgEIln(>zarvmKo0z}XJYbAvc1Y42Wb zol9HmE|~eWHG2W(E*SO#a0^Ux++X;9(3vXxO?JT^V8RX6E-Rm6$xwq?o_3w*% z_ceI^+S`-d1Li$o-VfamJqSGrZG<*Lk3o+?PeV^bTX`p~>i1yZIy81Oj<* zA>PX|us4GJ7|$EQ-U{~9U~dKc=|Svf?;h;EUA!0FZ^ORGeN%Zay5FXp_cE3D!XAyz zYnm~@{Wfp!^WD|p@z~CLX(#O&u%7{YJG32o0eS)23GIYlg{hZDT3c(co+}A<1N43fpY+U2kvU{_`L|f7fI{w#j*EhzK>W0Euzj+XepGvB6ANE z=O)s=VK|rYT;k`$_k8%S=C1>ZH6@A2^62Hywadp~?1Dc&a8y*fx8+!zB}Q&le7z|XCZYghc1V% zgsz0Hg|3CJhpvb2g6@Luf$o9+|Lnbgd{oDE_kUN5%&vrtWh{e?Wg#mH60$-RqQKTs zS+Q2Jj1?kd1ru8#imGD8rH)EfS0M_>8^xv$wQn3Azepa6WJ)7BCNWKD*e6!8s?wOG z)Rq&Hl9pIPq1gq?HFarqo5(!xGk5Op-DSm(#Q8pLzxiWdpWWH{an6}DXJ*d)GC>pE z0qy{I!v9XzEFZi$?`ppWG$}=?_)gR?AegL89$Qs?iQOr$wQ3|b|6Pt!;g_h_7}+5V7(nj)>|L6 z7~d0D_Bp8dyNECA@n^xa;5pKrBc1f=_abs`^S6Ib&V2YCq{YOloO_@rfAvkw|%%{-qx^2wtBEC9>EGO&U?E67v& zyH&|~s(e>5?ht>c@V8v^cYr))4Lm>|0rIE@tHC<34y*_3!FA-lj=ZIx3SF4D%0Cpe zj_dpt-4ANI2PtEaGMdS&nY>!TR~>_+zajp_k#z(17L(QBa|sNf5}D4T*BPJ&Esm#<59{z zO1VeKPw!7qcMu%s_i^$+0iFP3U<{l9C&07dS@0a?pQC))PfuC>gP->3r!D4yIbc4R4;Fy3zo`r?11rD^uoA2UtH3HSfNcn9+gN-FT8e|#eQsJ7 zYg$e~%L!;{AkPN!Yy=y@X0RD-1zW*(upR6GJHSq`6YK)Jz;3V`?1849%iCkI0(wU0 z*mTD>s5=BA|Gbzz77wz=VtLRSf1GuYHQ1%Kw1XDf#A5CtSnk?qv7UVv-(p;HNcwcc z#FrbnGn0D+GW9+S_8i2|JBo&3fRn0V9F33KnWaKhaCEC_p4?hznQ(FbKG$P1m4eHQjT0&Vh!>9ql)jeQmy zYpe`t@V49gEOs#mkTkMyR_+1h?iwqr2U@#ptgJrdy$f63hs<}`wmhqgeHJ11S$x+( zcMa?&w0n@xOU(D+dBfM3>ycmdCG7h4giW_?g3s)dc`W}M+nG~)7n(A!ztp+G%`y-6 z%ADGDXKEpp>w`r zeYG5)9N)*#-yczylH+{gOYgJT%|46uDf=ua=lpD2uf&HMBdz%QPhO?>S&&}Vb9}PT z0{?^Tv*^daD*G(Nr#js}|I?I%U)DO4jSp~--e<8Vd7s6fx%OGec=u`cS?s|l>CEf8 zHn`qCBkgf`+Uo4VfEVs!ziyQ9^{oAjdTk!W*CI0b_l~#XJ|^J{om6ixeOvZo6mnMf zgx-s>+u4he@JlX6j*fpi$!8Ky_V37lQm&lSx}aXCiEH~m?0Np2D@Z4C#or?LH~l!> z`ORs{TX6+t{HoT<6Fhs&kEpszX#>;HekJt$?|)$oAo8Jq{a;rgpDQV2?t^L8@x6)n z0mbKKzkAF3P_!xajv+St%p&}>b5uJ#(>B9yS90z}{hw`Ee|QW3(@tooXWI081ltIU z$3e$_LLQ>qqRS^~^WGZ693AI_Z5aEC9ansS|H5}=qxkP#&%X7Ptk1|fcjddMACokt zgwG({{RXj|Czm^#9laO-o7h8Wy6D?d5)T`s{8sYTnzVy!u0ZE5rwzo*K=(~uxAU){ zj4P?*+ZU#(=3c{ARj#GA#O>#o8zH4sjG;SGbcFRH*3*KK0oIcGy#)skv7Xc$+I8^L&rz#d~n&V|Wd zIyo06`{~m47X!TgjGxGi(N?0xlvhmu^^fpR{})|o>*4+SExoqSP>$#v?Me0=s6LDK zAnSI{I-86&Wo`6>|0L(z?qJQ)erMa!ZRJD$==E_Q^7-&g_Hf+D9*zRm9cP+R*4v~% zmHi;_ZJi{)@ztz5LfZsmWx1o{4Edj49gM`F_4V`YxyfU#Ef>FKj$vi{nMdXr5n0Ri z=hA;N7mGoIzlr~wkYSEJ*E_w3JvU{kVSix z`&tHu*>lst*|A*Cu-CJ;w`tz8tXgDWPnt%x3tvN6v-`J!_z+C1WI~n!Ezg z@@C$$tV(DJ5WfamYBVixHZHplT3&GOHiQ;6U(V+a8FWT;E4E(GfwOIS$+$=EYw@ik z4|siubt+kdmAL9&wnAjwOq^!QYQ+Y%2Cd(dIN8LJaZzC~!nh(59cS){ZmE4kq#d0T zv^+V%i0t7|`-bM~@elv0yZJ-x8(Kxa6_i~ORCdtWyYaj4WZw|+qN|}lpZ~?a-ijW| z+USG!ev%gAjNZz*XxfC(YUE3Q*TnenKGu9^+ItehDf_&_8TLM}mFV0f>>FB+{;dpJ zLH3UQ=~u`HnqzxdPhLa#YQs8>++)xn^)7PGsznCKTIwgZ6ds4!H#D1g$RWOu06Ls~ zL)&OyGOm++Wxduw2c~{RAIfiJ%{<=bJHGb01RkL$x|s48Ggf;C?X%qe{x$i&(3at7 zh%s!j$NWEp#n(Ow`)htn-lN6PS4^2zlu<=~)V1_iY7Z0jmQ8(~euO$(9<;tPe8a(K z!Z#c&En+Xr@D~rZA_r$p{%=d|K2@#B%i3q9q3SEElDsRB3;O5}AeU->2lT$7GGsQ+ z-D-vWmkH0ALF)yFXFv4#HP6o4Co(L+Myj>X`S34wj2#r&e)d_t_6bk2pG(@h^p|RU zz69NnJwqZh%8RZA*9O&|p*->vepMfhT>6nqeC?Bbe($GZ_q9HHk?ee`74Rv2E-@%ovhAh<^6*)lE9K?RL&O$8FnO z>OttJnMK}r5_cB#2tp6_oOXZXP!4f(pr;}1T9j>LPsqE(wh3+2zw+;8+wxMjttN{+?wAzY797>Kjks~! zR+CLRtKcDn|BiW^B5z^Z?OULU(D<5hn#^NZ<`c- zmHm)Y#ETJ6+PCb56y20`+}FoC@_pE?d)c#L+pVPa*5R2WVvEL}!d{V&*u;1rTx{#` zY1%q#rV+{enee>HfzZ5z{o%Rv)%P$S07Lii`vyKAKKP=?c(0T#-+lNHf_l%G`i=hp z`Q9D>AKyWr{w;n3$;0_Citk9iV?!eSl*gI%Cw!dK=nr2%d1%x1%X-5fVw}Eq@@dN6 z6J9$h_YQ6U9pelA^sD{s8|r1xmh8)ty;ibcM)t}KupdVD%aqd2`@QQXDZ>&QHHu7Q z$alb-OZ;p2d{D+R)>xQ*2G|xw2XVwFEq>=eJ5D>^MA}WZt?@aw2Hics#<2EM7wFDi z(T9x4?tNJ9V*X(~L z&)N3>vUfrCmm%oFZbn<=W7~8OdC5Fm%SPI#qW@>&JD;i3dr3c&yo*V{khJUNW2Y}7 z{d-*L#g4_WuhMVCXvflbq>iIS52ZXuSLD0MZ1fphHbVaizK^cKKQ%lv$v^cUj8CT9 zV=8{F%k^c1(#7>o|9;sJ*DzO?k`LhpgOtQB%`@%t_B))CMO#>0~Y=b$BB*N2v$eU3^b!Y90?e zdHmU2GpO@W^9<6LF>iW?{OmocuJ@-y_Ff*Dn-seD>Vz-!8Qll}QF!#>Bla1-Gd0@I zDLRaKwk#IRGaoM2vbb%Yd6T9?^1stb&sxUC(m(B_|HeG?YKITf8-5u_xO|W)G>Uvg zhuITpmPi{xcZ)S9rPDmC`BZPVYs?{YNFU>k3{&nB;vNy}B<*N?F5?X9;Whe}GqktU zvuP7^=^H3-qL96t%oU$yuBi4ZiftIa@nE!A>@9bSFg}Sg9+5sv?N=4~u~${>p^UNR zT`u)T8GT=WsPJHaX#K&|y{hsqy@X=}bLCzV+K{qmGKQMkd^>$ynt7rzJHeM%e6{wO zxc(Mn+P$e0(D@p9o`KKP3mHej=j-fE{VILQKKhbp*%SE{?D;d;R{_SX+EE+S0c$W(eJazc*L zVv!Se7n_S5krQ+HP}4MWYSMC2w%L{w>D_16rn4be;{TC-`J!J*dKJE~&B&Pea{Li8N8n$z6)hTMfYjv%%drw@B-V>J@2qxCL^2 zGW}|cEvtc0#)$Z@xO?8pY9x;acyf8d&t)o+PbIV?W0}jyT=|cX zz4Fnlq|RLRrP>j7bgkG$Xp}o9&U<}HW23jjru{;9exeWe>+_J8V+&`Y1DCR!aXUiZ z!@s~;%j4{Yzo3ke%jSHdJiW~cV{<~>6IxDvP$y#{Y|e6Q&O_K7Y|sSt@>&S}T#n6o z2%CcqdVTk0*qmaoDSoDJip^O_eV3yXwbXwt-xq6}vl!hVJZ^J_u{l-P81$sPig0wL zyocYk|MDJwcc-yA_o5p^hPiW+JEwix=8$KM^8NMbBl_p~B~?2v)@`#`)0C_2&K~Se z1@X(!-wM){q0e3XMyHefyE!hqvo>gLrre<8vvtaCB%QR0X0RFD2;Uniuh1@A>2{&3 zd_S@cr?Gn-q{pTt`E#Xw?#%LL+V<8X=hhs%NBK#1Z?_R)F2xxK!?t^euzSd*N$g%r z(7K!cO4W-;+r7EiGm%>j^w!W$k*hO@m2uaf!>h!Xd6tS-2Yq6vnuy;-e00}s*T3k* zpQ+`%nkfjvui_KCEm{DIhjV`@{dr{)FFt z`*Vxbh4=EtRE%@%aU=EeAhNyUbktAt6{K4%x^)TK;&uy~{#ECFIb*DNI%xXI6{K^< zjGs(Lo7>*KdwJa6ab`2=+;5b&F8#UKr?Fz}9qn>_DfSNipIC^!E5_c@K2I-YK7bCs zzS}TgU&DW^4eK>@?#xQVI*s2ywjBLijXvHA?!m8I#=1%wztJTbN1P6GPSB5TX3~cJ z=xHW4McP0;zoi}2>otIt*f}|;y*#MatOmTS(G-wZzN{C`U6z&0xHq5pMWijV_tihW zY3(vEwu`p^vfMRTu-@g^DwZq;pkrlz&Tu#A$R^)vCQ&$Qc@#1nsh z3_6S|Xl!LHQb9ay@hA7% z@w+x?-3}d#7)!|AB&W;yFGoJu3uNT@acGP7IFx+FPn&w)HhKJfRibXgS5dd~@R{ec z9(a}cE2~?0ngu^ebxXbY;U$ZD@WW458^2rm-Nx_MY3g>E^+05$eC7eY9$0ByxCY01 z;AZv=#eL?;Md?^0Wk}z$4tmyUdNQfEmNdH7Oxz~uX(oP?iyrZrQ^&G~pnT?Koj{P;0fJ$Xn*%CF>g);cb{`{8FdUHa~aes)}1Qfs2=?tXCh z4ZU-xt;cAXz9E~wp_smbe&KS?kiWi~zF`%8!%F&wkYSzPLf>#Jdb$d|T?#(w>Kj(- zz9C(EmG)3adzCg(r`u~t_YK%@)n3Q3AGA@`H@s>04H@XQzlgS0)MJ!(D)p$^9&t}^p--TEgZ?0^S@#L8TO+USp--5J zy&%u}BI=2_(wC?{q1v!Y1K54~gVF$fLM^{*_>Da%tg6PLGQNTSelhii9efSm z&LEG|8<^iL7XRO6%x^->Z_tk&GQTMlKBxog&qv)w;YW>C36nctlyAdB{^M!p7#j^W z4$j(0yf(Ye{m4Pa16i5W`Fd=$^cl$RbBvW{Ay>7|O8=aJY~wbT{ACZx>whfcNj(pu z{1|jfU*TUz+$QL2g1-4SeKMw3bkl#8QP%0G&=>c;+{+k*JLR@I za}df*nuC1Jh;U~3y3q}WJqP)=5qZDpAo2XrT*fCh2OShT>!1(4SM!Lun#Q^8|NIAZ zQsO)BvPfL|3^kW%BuyiH%+~Q|vrmM34P}pl^10BDM7P3(QL*KOYpzs8mLb&aV@kMo^z_l@b|hFpGz zYbp}9HI&#d-5=II_~OgAtH11>2s7ktdNy?^bt5)Ke1EH%17mx?-A|uU9<-jMKE&Rg z#>NbUroR*6Tg1QMP6RjKZ@fFscal#x{LWwOoO36(x`Xk1JH8KW#G+&PbZLvsU#-$+ zbdmEoa?bflFZSmt^zloy5BAaf*h3#b=Sg4&&ikez&YO- z_GgIu&ri|EE$4pU6U<3Yk@p1cWI~_yUCvqG6SONmCz-t0_`*lupK^|K8|M+WvDcro zB0kQFsPjip>bCo&=0oVEeV(R0WE~~{BiNs#ly!u$;~3`yPH;Y8jPn5}^jRBhDXvpQBP41`LntIN8Cp7GYh6!*2v_K1-LKaiV;w=7?$81@M-B5jq)Zt^4 zD>BJ}o-p(bLQe#G1__Tq&j|D!g`N@UIhvrST+TT|k2>cZ=Im`fXaDNt>|bs`-j~Rz zqwEICZg9^2H3UAgjqzXVIp;p;>?3Yw2A>u5XJ9Uv2j+oA8H_zvSn6!nBgyo~d3Z$g zfNaz`=L7JtA07@6z8@Zrz{6p9I06rcT|96{x#EGkh>de@GXM`&@NkH8sul211zpwf zP_215^ua&{JoJ{P)c*iH3=p>jnoFR$608KPz$&l?tO0A`r50Ygsf+N1yeNAfcIgQE zrp`m}gP#HT*#|!Z@G}HIgYYv1KZ6N=Zst6+(vi%db?XnLPp?6DqdgFDl1=>W9Xw&MP^ImA~gXX=2_d#<6nh!v81ey;dXfEfD za#aT&>L5fNa87ns5p_^V9pqC7MbKMD9hB)hDEwgH?;hp+$BikxMc^$$+%9gw z;vRth1JFMN4uOZk!{7)w0v-d8fn(qpcnUlPawgBmnLJ+u{62JXew98xOXvM6?2bC~c?4dLz{@#kI|pqZWamL{nP4WE3+94(U>;Zm z7J(&T30MVIfdSWtzKwe9Bkz>6Vt#lkmov{& zkN*e{Ug~j*`I1Gy^(y_=S;oy4={?k=N7v)49|$a<9*vnP^*93WBk*2LIo0q`2iAf0 zU_H1FTn9FRO<)_?25tm5g56*@$k{v~Sy`&pfL-eb7~>W$?cT{(IoR8~%Hs zzZ?E{lJ5@mcqe)9Na(RT_bhe#L)I#((+kc$cjo>$lq@GN)^JO_Gc zS037x^s{Rcc_?|WvGrf^Df5iSnZwJO=Ut@TMOr`l???Z0z#K3i%m)j|vw%Fu$a7V4 zo+{r}LF*gHBc8AL1|;9Tbe;@fbkWT~oG=hy_GuRBaf~{aX*ba73P6y>kKXPmOa#UHja`(NPpY@ub z1C)1w^18{doBVpf9?%3$a0j>p>;wD2z2IJOKjrSHT(Rv-rY~2Oy(DNgyUSj!%O0fc zLCTJhXM{Wl!9nl{cmzBO9tDqs$H5ce2`~o6zzJxWfCjO*Zgt7fH`13m=6|!1z;Ii23CL-U?p~;$`@tLxHd zpp}f~Gq;7N(K$B#u?_kh^!i}r)Kcd3TY?dF4th0qYAx-)1$)1k^90M8ug?*G5BJTx z&OvhqTHQAf1}=IQ`dBa`zISyNx{Y(iM>8UFmydcQqDkbjB0^sl;jDXNKLvJ2e&Zuf z?5A+T-1{kR zKg=9S(!|2on!N358ST*AL7la87TQGa+p+H^vfpmo_pFVsv(U$x+a8B@4}HZ;%n7|$ zneXUjK1hDimzXEI?7XTI?2hs~@W0W;ev0rr-}e0!>TW&n<*jF4&N`vvcZjcj(r>)` z3ie*a)6sAIOrO)coIM*?Lfe(ZjnjiY|Lpf^AHM=wT?sueaZcgwq(S_@QLn*1FwVzZ z!9JFFI_k;Gou@+6mGJR)$BolNJ!P;ib%BiBzMs)$>3u)LSC2%8eP(neee#p+u~|+V zZNdM=xaTBenDMXSgJv8uA-0b((3yJ~=Y0+P$5`a`Exg6cS(XaUvfRR*-Q}ERS=4|J zh;urnoM|cLtj;KJI*x`6YaL^?SO~eVWQ{`=g;#`G25O-r;>e=dwO} zjULN8Wmi#l6=gSo4PX;AG(kfVG%QW@u_7m++E+n}S&Pn{IcFh9=upEk}keVj8u zZJcXLy=SY=V_0p_aSJ}6k3+}nkHE{g&%8R5v!?v_+96-y<3qFMK48WIluf=$R)=m6 zc#zd-Rtmk<&{<8~X0RFD2yO)1N!t$HxzN2NLASE+OHyQ*120~Yp&!}#kRfz)HmiIu z^6TTX7a8`sWQf1WD>4l8{>3ZoYjWPVLME#IYN4SP8alyFum|h`w}IQh?a;Oz8Wtzh zCG{%jaN~8G3ti*9(aN3ma$YQ(JL|K!v%Y*6H0|WG3%YhD=qeWCtWe&JG&5&5L_-@+^5Wt@RibDf2T&3h5N6(M(Y-x(u{|9B8NNgA~_A>+Vd-qS5Z z9%Y2plct_DqUXi7Oys=uzjIba(#U>$6~B`BmBeoZ8^Lv?TSq$4&G`vg%6d(oySzeO zUVwA~(zSxEU>n#5c92g8`G~H}N#vt+We&dYL|%c|Y@JsPX=_N^1$KemU^i%jCb)w< zcTmUKDfwlyM}++3y?c4Pmv!ApY$5SvEuS&n3=ej#IOWY=x_3Ur$3ALQRvs-$oVJa^>&SI z+<6zObFL|QC-r~V#rr?@P5h7DkMmUczs`P~KY*5BWmh@$2lT< z#pV9_a%>oO@c+qvoEH*n{z+?Yq3cItEtzJl1v{~Y@z!R>TiA^8pEBNhknz?Q`lHSC zLD-GcKgBLRC~N*+^IZ+BxnZ+!sbkHpmVT&*@m95Am14tXFT-d!)08(0-;nXvdhAXM zd}}|VoNW}_B4Mv>AiRz~#3N&e%nSXN`S=C$GOVn8d;@ta?0Xt&^_p8XV~;N*^Oh_# zr;o8#H*0RY48|l`0lD{Z#xC;SYw&T_-0lc?vB#qx;|bXZIKL4aQ>e!h&=P}Yzn^)p zpE+d}KC~))XhjJhn)uHYA3|@Y*j(uCA^&d1gFTdm?a#`E-W=%7h2ES5y=u$_y<>ZV z)~i3k&*=F3opSM?tM}xq!D`B{qx?F`AEG~6o9I7A*Scu%L&Nxt3*WKyLqjGs_@N=w zMT4xl2@Si0)?Yz`F)xL8)+hY=lwA+jgUrSK%)_0xqJ7QKvL-=`>L1p)=*fW|`i86y zWYUgII*>^_GVwsq6mxYC^i0|Bv&-1%W*I{XJzol1!!NN0I46akBFZnK{8q3PWWFzR zeVO~K_lvucQ8zNerbkyN(-qgR)zBdIl?!cqplu_x(dTDvB)kpUEb=-BZ5H|B_mr`z z(5A*w&^GZDda)r-vr0`Hi{;c)L z-EGRJYl0@fcYr&V+YCRggg3*_S@N5J zAMxc*xcHH~g@vCd;U_f9%sBx+$Kj`NRv>ExevXrV41UHmKZSP&7RjA7&bZdm(@OHM zByY!O8sT?@-;Q7L2)|_w#PO$%z}E=t>?<$A(@M=#K0LL+QxiO`gQq6K*TGYaJWs(> zjC@bIc#^d?;puUB@=`|!;AuZRy-NEZfT#VWkHAwz^YrQ+fwSG*%bTIkKEab3>o#Jy zWN*q`Pvm)Rmkf`=c#J&|YdsOMWk+NTM0}ZtGj&d8BCo8^A`e5o{)pX7Z4BU7pB8wd?Yr^>@&%XeiYA6p*ff zbZuZ8xDmw1=I;bM!7lRaB0p)5#mV`pJd1AE zcnmxS9;dA1=AnKTzs;pW@8VRQ?85yZVg)RWGpXx1D$i2 zLf4sWbh;-PkvVJj1bTkV6Y-xiEMtPacGBi%qPyr}NXG2MmvP1E@mb~~3q3tYUu&Va z=Yrh5@3Bfxc)%L4#@;)AmiXQo=IJnTP5GdY=&0PO9({y#E?rmhLjGzjPdP>-e$LSK zZ~It&+1ospcgFp3{~&GWr7O69FP@IJGjtj0;^TYT$!|+r_NI*Q-;Q|+JX}M&mAtQL zJ_8RKzeqZGnJa4!sl2%7Do@OG%~fVFSGk6LqPd(Mm|^yh^L`Y1D*HOeXp1N3GG<)J zn3482?KyGpyvd1`_;;|4XOEno({5O?=OAA@*i-2N8}`BpZc-Ax0}Ahlzq`jdvzDm z1{YEW`AgfA{G~sc+T6gN#t*WOe#V9S=zDV#br6z$^mC{K-ce-i`3vp8W^C~b)j=jQ zhwiLQWC+b!wfwH(cP+nbrm2HHt~z*=`;OAp!JGFiyM{XGb-nc^W4YfD`Cunw$RAs3 z>qdz!``<EA~`$vUl7Vna!CeRe!`0`?nxq*OS~cAbYPDR8kk> z%W5(r68~z6Pq=|jt-}UOm|uq-an8_Df3by>nL~LwN-rY*JR{=E!OU`D|H<>VBF?{GWyT((4Stz6xQ8~ln>M(VH({Qlj=#*@ zY7c$QZu*X;HIe93ayDy*`L2!3+uE47wK8vOrY{*}-nI@sZG%?&hSB1g=IA4a^;QP+ zwnvcXWAIx?Usy+9__)^F$Eg#-7q#m1@9Z}zA5wW)zI#aBI_ZK`R}Y>Y^~B`X7ZGGZsOyV9hAF+{=S*9QP?hfG^}Z&Y&8dyy&e(j z;{duCfz|`)Lmg{@tlwtUu@=Z0Zk>n!DO=6KDBI%+S|jlHLS;(1>*)K}(f6m0H$0TT z#V%jpfI8unAhK4{nv4f~)$){)r@lO=1(?1zcdAagKi82et(dJY?ma8GxR>&@({zUg!xxPZjh8pr`(gI~Fcwq$)poOwD?iPCwcV=&?_dl)x-f-qXH4jfqZs$A{Xoumim{51 zaf%wRK9o#kM*aZ)LgztifN_b%1KdTZRSo?Qms^)6K!*@l0Tzk>Z zvxg}&&0e(g9|*j;mHIDA;h`Ox+mZb#@Dz9!JPV!!&w(eRM*P zjJNSKr0%atePh7cC*uo1Zw>T5n4aEv9XtqaqDT4A?1g3%nzuuPMtBc=%D$64_>{dTc`iO>-=pw( z2tLPi)Lw73&-+kLK=yfOoFe@MebI#UH`L*w`oKN#`B7)Dpu?v#*Xe-24*2W_yTKl? z2fW<9K(b!3;o|(pdETITE`sM_cNH(g-c+M(ke0?(b@`jcP7WKeOKEN!~Zv!5$2WFYlD7ez6|;)g!;1#AjZt z`o*<6{@S3`Nm+7lp;O-~O(SUj;8 z*ZFslzJv7Q+ZDfV9+(Fffkj{mSOQjpm0%U+R8fxD+r{b2QDrR-S`jxt#k#yM%Il)M z8uF_lzgn;sYyca;Mz9fV2AjcFuoY~l+;++p8@O=#a#h(2gVq<_WzW}T_fU2ZWp|Qi zCwX>(UEnrw8@L_Z4((iH@ZvNmItqiI02r8p0n7BxtBr9+@O^YEoT-6 zBXUpD$(8tZZpE*&gn3;#dv%N1mp31Ka&OS~@2pYnIB?o)n5#TSq>O!cb-6}B`Fm#D z<%u4hnT?GZ3`WG4nO%h+vo#~)uVDXLRgLvKJ7{y+w!i1w*W&LXzRW{Tw_IZ`s=>cg zjen;G|4wy{waDYON~bcw5V%6$6~MThdBf?iV2cjR$DZ$S&d!Q{4LjwsQA*y(_7CvK z2wmJniod7mviv>2PnrvT1&K3j(eRbM@$rY7ph0|OGvOEAn|quyW_%w*#^3oJeGewG z_Wc5HeEfChul|jXr@7lx?thm%{G^}n`yx*}L{c!Gf5LxkE-?^QBT|8}_h117P_{2PBak@3^OQmCub~$ez z2;HMz!-~KAK|gdk{w^n-+#7g-9!^8smE?OlamAm2&Z>LcKK7k7zO%Q1j=$YWJMdoI zzZj=u$YXv~*VUEL?=gSnZ(2~Vub;PN^s9Q)qKLM1DOovsb2;+4lDP3YXg18l>3qT= z*E!$kZglx@1_%CawFe)beP5&h=7;ZknZ3`~`~~N0GM4|~{LJ9U`Opk|O@0IZm@UTT zugP!2x6^}fryJi6`)$kZ!Y~*9{q1_jg@tLCgbs$;7V|%J?HDP{jIXbn^Qr57|t_uGBd2tjj;yr$za}> z7m&5p86NWXXYg5J6?y1W*EiycsIgwq=U#uKp0h-4dL9lvF=&jRf6!x$ThMbwU#5)T znUm%~pC9^ipwAC|BhYsg`bMDdsEa=CZc+4=Nxs*aIn3v0F*ZDN=(<4GDQJ^1;yLm< zS7Xz5=WQ)nB2mjQTUoTJ_$-4E_xVcpWFY4k&57#cIl*8`22l;MHK zBhYvl8jnEZVHb_~gA|P=&=`V7#=x@}51%Q7#t~?gaq$>=jcFPSp>ZTZ{nZG%KtU+i!46R3@^{A$G%bI|9FhT1fXzg8>LTe0~V$kRsuQ_k*IdeTZ zA972xra2dyL(qHIcPo$%@Jrm0L>9-KH#ER_Ctz}7Crr-?vwMAQ80T>797=VYp@Gt-mdtE$;Ur6K+@3CRdRd0ufZSb(0bJgAOunitecrY~&yBh-D z?gS6-5&7Sq!b1-<^+4M;a2vQC#-9R-hqBj5;l3_J!N2akg%z!TsYI0l{qPk}Ko22aexm5rorD&NjA zcp8DHWAHQrPdnjh2R!YBryVYy#J3|nVGCkm&MUXVQ!_lRho^P$)C^B;@YJSxS`Sa_ z5lc*>hW`?!s^0p^1RU;$VJ7J(&T30MY}ffZl{SP52wRbUku00XyKw+w0C zu#w7ta}?f=z}r!HI|6Seylsa!6W+GFc$2fO!W%X@HjZDo0p9B2?GV20T6n96w?=qt z)Vv*99q`sBczY7w7C843IDJzaG_^t7MsOq84z`0GUe&k4#VRRJRXL}A$Z&dk3I0X4IX=3JeFx5GvP4=kJa!PfX703 ztc1q^Jl4Qtjpnfs9xD?(epu?yS%YwRJPb{Tq3sBG1Uw2J1xLUU@ECXuJPsZQPk<-D zF>nk#1)c(9U<{l9C&07dS$OT%yjH+#3%m}(YXn{g;WYxU-SFB4uifz4<>IwM^Xh@u zJ@}Z*;I#x^x8P$gfY%aut$^1G&FdEK*eFoEJ}3MA3Z?y(r08l{COTM48n6Pa1S`QR zunG)-0k9gZ25Z0?uokQZ>%cm&9;^o&zy`1pYy{VV>uTk061#m?!ZY*rtOM}8AD$1u z^L}{lgy#-;?u6$K7tfV8&l6vPXYV{S2j9=E2F6#f&JASMWmp*v^quSQajaWm`$0QZ6W!2RHU@BnxKjDQhv5F7-D zz#;H3co;ka9)aIx&2JU_7Q^p8_#J@XeegQ~zwPk55q{g@ccY8nD$Vbg;kRY3nS&2( zRyF)ChhO|^8P)Jx3%~folz)Bs+Q6sacl7!ce*IT5HvBF8*kBHr3+94(U>=wc=7R-b z0ayeUfhAxGSO%7X6<`He308ttU=(m!gn8h?}hI^ z_-=#mR`_m%?^YMz0nPUw_#VFA%&CCyGWgzoeITm{zRTdd622=n-@88)_)Yj;Fgu0s z9_Z+So^9YZa67miG(i*G0qy{If;+)o;4ZKa>;w0Ld%*#40Ne-e1NVdb!2{p{Fak!v zL2wWp0*BzgM)O|{|KsfC+XeqS;eQwW?}Yzm_-}&$X83P%@n5a^-wpr8@ShL=dGLRp zc|;EUi@&u1{tGn!=NkiWJp}*GJp_*J@{_-xymP=DFc-`P^T0eXAIt{}zyh!cECNfw z60i&`11rD^uoA2UtH3HS00zKnuo|oZYrtBt7Ob0T8FgSc^QjtSFpLa#AOjN_>_7%4 zGFV64Mr5##c#R1e++4$Y1O3(5mx5Ny;qy<$_ocv_T5s5HSpUH~gPOOs>otTm27Md* z0E#K22l@0MpD~a5GU;Q*_8f0?E&pxL;DZcJ;+Vu4;$A0roTbFsMVwv4=>z+~y~N#1 z+&+)_ygTl##63Wq1H_4dZvX5MX@*GC?J>XRPP3de$B283G{-@=&vuO8V}zd~?J3eW zd(2*U+7M~+d#dx{9>zN!#yzaps5>?Mpr7BYTdDKntYP_BzcOk(=Hu=>R+2{raacD_ z_lCAVKzKF(SA#WR4OmNFwd7UgF+b(bYZZC55~r0oZD1R?5!?v2gY94k*a3EeonRN( z1$L8XH+gzI<_343tI2aOarP2t02~1Kf&0My;C}D`cmRxm5pWP31c$&O@GyBFChsxB z{GcoEf5Xq}pCHZzan6Eg!E@j_&;l)R3Y-Ex^wA#r>P#>b^n-pd2h0I;XCU9()IH)$ zU1hu}WmFTVnm9FJ4Ok1-f^}dWSP#~N4PXP<2sVQ2z;$2~*aS9%&0s5KwNh4}VczH} zi@gWfFyibW&Q5SAxC`6`_JMuiUT`ls01klrz!l zU0^rZ4fcRN;5Kj@xEF z@Dz9ojDaz50-ONPf@i^V;5pC&EpQ5)0zLF89{QHdt1MsUZ^`=!=Ci5vv_MZS@oI@z z2iAf0U_ICXHh_&_Be)J+2R4CCU^CbZwt}r-8`uVJ1UG{1U_00Wc7UB=C)fpcf!)y6 z4P7;yPfw+519TlG-eKY$0gr%3!K2^^I07C6kAcU*%3hDwpaa)A;6!B%&Chnftz?h-lT+&$5pVVz$;FJLsUusp3RtPE)IwyjWad3@^*)~m=P=FK-{ zP1NXw<}PUMTw!H3An$tm^9E#IZ};a}9V?hmvk&`8>_$H8+WD+&d*J&ie7eY@=W%?n z&y(|FP_1=+zNcfHS7%>?tj6%{s+_Db$KPe?HMm${<_6C)2(rY z5^H|WyWoAiIn~>OzZ)C52fpO2@AXgE@B1t;7JR>#bu!s&**{!xP~z<(j_}(*+;otA zkP+6a+lMmjHM?Hc({3iMoO^hYb6P&e2}*8^Z{&SeANvV>e$hwv+{*ePW1PR*Zs&J{ zE^}&g;?4W$>R`lIV6->8^3Rg|m+1V@>HG`He@UAB3wg^y=bu5k80&N@Pjt(j=Q()l z4c{=SXm;|P&tBBO3b%_kAZ%bl9EesN~6eT1k1%B~RW~&N*^7j9p)| zh0gmfq;q!Ax=ZKzRnqNn^7JR?*{A5#d7jXDW(%EZ^2`oe)jH3eq}%G`iJfTnCG_fZ zica1M6FM1hD!rO1bnTKr@5uM84+b;Lh{bx4EC3MOo`(oPJh^b=Um5am$>piTYu;y2k$rRw@vRLf7PbL zADQ$ubB_o0AEmC9jHx%-(@^R)z9DfWypVoF!uq|uZ|7al+ii(=?tC7jJ^JO{H#m9A zo^5#}vD6b(w!hS4MEVW;oyA_lPwTpuyd4_k?Mm!}DyJWs3NsD(0R0+sH0El|vr*p1 zxq&t2F>g@Cm$ARxM=Y`^WZZRrwtXi)JVbnzM*AP@Hx5tF-YWgb8)Zf1rcd6}>Re>M zH!JawFY<`Ii4IL|zKwm}o}_)=LT3zpi&7@MO}nnYll4^hnRTH_?juFU0fW6KQh!0h zjCbVgeE}(FHio%Z7#l5is*m?zReI{zmtoh7{FZ$~24Oj}Z*3uVmhXM~yXb;1GiV8) zrO4tzp$D1we$M#9(|qK8S$SKQx;Y^AplI2eZ_fX1;|o8MwP0*GZ{EuPN)P0_>=B~> zy6$$d^+WUHdxlOIBM)pZd;iTv$fT6|Q~5o6gIP~~_j)Od_iB^M^3695%IbxNKK>i- zlJE1N>7L2k{aW|Rxx1ME&whrI4$ogyOntH>F?;(*peMPF=bCIRo z`946o1H8K^?=1R}kJQbyd=MusAB5ka`S@e%!MJVfD{>Fnd$1E@*nhcqqm}QxF=71> zdQ|)6dp~D+#3vDDUN7g}2E6x7mS%bgzh_eE2mJT)UEYloyoR*$pId*T=g+_4+rLzu zUp&7Qy~nR0`Haqgp?X2**6JwZ(o)M%@=ocild;C?L8j=LAImRoNbo_1(8P z(`<0I`-Lj=2k4`bq2FC5uIyD5f1#?U+X~D8Jo(@^Dq{%vk@KlCM&X=XWCD4t;y)8Tt2KSLPj+#^xO)P87-74lu%Ceb@|>TGo8?iZ* zkkaQjip(|WM7%zELvw(>*tz#++WI6;+WI8?&*5FwlcVd5AqKqUiOuunTueu%$&<9E z&QtPDJ+CynGH88{|Id5nebku)Q^WXgv|ab*)L8fH_OA2`IrbtmvH3=h=nHS(5(k+i zY{pM+Hm^n|YD|fJP<8Nc#pcgBFCjL;)mMUt6kp7+ig^!2& zpCk|YU+Gg{mf0`5(RQ1%FA{Hv-$e_u&8xkl^Exl|L*+#q%ak%`AF2%6hYF`piI>6H z^i^F(JU`x${ZGQ;Z+X!FsdUswJkEky=2?kDU#arW)^X^c;&FOsnNf)|Q^%R9Nba18-@y*aSR>H9hPWbJ*4_5J^)d_!8_W^c&B*JNDwyl(Ud4!dKW%Z)`ktmk;!EG%Prhg9yH!5ugi1?4FE(6cD}7Wc{m24z zTl%;s$WP^k-grD)?>a<&V#i16)1+-2Ogthk7*xK`h(s4mU)`unKmb9 zJEKcd#su*EUWaGeV>3Lrh`fo{Paoe;KkbY$r=^iNX=x;Ujixb5+5Pmj{q&bE`xEh{ z8#53mZGMC=*ZJ9cUeB3N??v};)`%{6Q}zRT<=&v{*+VuLTbr2}r|UYQO(Q{y1V4vpM~CPiRJ~&72Z&RaDPv0inVY@#+)1UwpDuk! z)^%H7DeKDm68}Hek8aBR_Hd5q81piZ@tBekVd8&3B;n}03a1>W4BC|0PnkcVP083r z`t&lfANT^I+Fzs7osB+4-$}Tnm3em=aVa~if71=i`q^{)Vy?aZrOQ9d{_DF{Ij%lm z{&U)>>P!Bp%nZ<99rE6A@DO8Gi9Zl#ED7(jU(2WH$vVUNh{#EY9`A<_`WozciG(Y9 zRs|!s&vME~wjv|)NsKK%_j*9`t7_V>`VMM zj2EhVLm4AtQ+mBmy}y?*)jqHd{o(5;_o{rZhwcw4+aGV^`i;$aI5t=IDWm_d^oR58 zdI&d8D%-zY$z0oV7mtK19+zVaGBl6j52-%1pR^hDHN6jbMuZpai)|;7r=*ep#V$%& z|4bdrdoTS?T4;6BemY%RE$0Wc++}Xy$Vtlj2Km`#tfLI{i25_dHtTfddOp_A{zBK7 z@8zPOmzmGI+Mei6f-j|`XSbzED|+bCotKM#dy)B!&ZmFVhnGF7$8v{gPl_&d$na#E zpMlm%Mok^KnDf6Fnok=^Fb{m&d%~j?Xg~qIeoF3I}z@bse zi9T1!iTKaV(sH72O_J01%gx&;`}}VDEM)2yOuK$Y345O3()Q%dWf|*@7H+K`B23i}eT?)?e=GS&-{jIW=_7>Rc%N96LgPKR znDe1Ax?Av`DAMbF=2{!pit@}@Sg#n=5I#(r1Xed2KPyY^$0-y2>(DdXQU$_+8Mls((- z_w3KR(x`Y|XeN%q9Ak`r^wYdu5Cs)4EOB#* zEA$90FKQVIFJt`v9`kw0=K@~B>Cb8`5WZ8#1Yz;pWimcSXK62E=&ig79Hq=rbhf`$ z>o4WSS`7Lr=@V?fp7ad^;(ue#HULdB2PpNB2fFa*-^6c2nyJmTj8(MXY8tw}!aCCh zbh+w7trvs}51!O=cqX*_AoH-7#b+k|GDjYdi(iSgkGM?f1E)6o@JVLG=an+=i+O#| z2;Jhd6MtVH<2~^=^o3>2*ECsJApSp3Kzw)|gxhw(L!Y7NjNvBk(ljhdBl4HAPqdh} z(VbzI4e@RoV<3q;^w#;i<$q;^2$wrgxD!>SQDN9*nRkBP9ftjpdf=YaM7-{q=Ktom zq>*uqe{5#Je`;m`8ZCam;>Ukt)cAy#yLP$K6*5m2*-UM|9p9lnFA_S1-d^hQG=7U` zLN^^OWBgI(@tu)7i+Z(?=I(DDb#mHOBdkbb0-=4o_-3;q*LwHY*LkDdqOjRvmlW7g~RCi=n~lCDeB&v!Dmsb#r{WtWuFOBaa~`w&=x;K_zTiv z{KQ!7x#pIe66LsP3MJN(9R8lhPK;x7nf01x?FEh4Fe2qqtVLnZm^Zl!QXJ6j?#14r~`#Qg6jChsU zF&%$IhdFV?2K%mIj753A>*$l_@mb&r6u!d~P&82Ae=li}iDM6A@Go=lHa!P7GPb_* zA#9$EA=Fs=mQr)kRr-!*WYZ^O_64=p)8LBFah9LBr9$IE;TauYX+&-k8pXdtd|Bsl z!dDZQJDwH&++{?$#@U#je>psHFX9*envcU%@9=%kA9nu^-}n4K=-*PGQ=97;FX+BX z%jK*Se<5X7}xgfkaCTgvYlB1it0^-YnLEsMmO-zd5;^w~@IT|cKBsi!1cJ}frLJ~!y}cQTf8 z?1YSoPm<;>NegdDX+yR>71~RoSME*{{TiLS`-Zp;2q)XT7ttkO-wmd3=+H9!&P#=# z7YH+YZ!lk^{(^dag|O$0;0>n8U*>oo{7XmipN~O|5dUvDxsH057$d!*!spd^0QroC3{z|- zHe>MQ62rWG$&+`L4uARg4bQ@xX#aydky}*BEjY-&vcaS}yo|j2ckaCF(IYxPUB>;t z^Vz%Ze?3vgR#gTvka9efbDJ^vmxRSsxy654R7;qz=q8^L_|wt?BUz8a$vWcnZ_kom z@=L0ll(>2MZ#{K8V;`UI#V;E8Zj$1X9a z6aC+X<%;|hd5A5&YG8_anqy0)FO>0vHd(xljrKJ0{J zPh}n~Wqq1-whkF*+H}8YN|>~l8p9O%%G@vBry%Pw?26JIWUUZ+`{XVh`t>Yt!iVmo z4rD$c>67fwu&XYqGoefN*$GciQ(o^LW8?&B)cApMi_GDjJ5FTYA7lI?_TA9q2So$4 z3O}l@Nz2$;jRk(hcd1vUm!$PI8>Y;yV~mZ&NAMc$z+Jyi*)ryl?~;CMb8>z9q&{`q zP<@Wr$s(7X45zmv+1Td`?8tD!ruHEV$3BQ&iA<(zA7jERdL;R&`lBui7r1E9^i4a* zNYM?k;ZonXh}=mlvXplAu=g$I9JVZ-KBbK?p;O5e-NVi%^)GjN-onRm2%m<*d+YS6 zU%0{CMH=^9dQcxLpWeC~L+k$^EEKH~OaMFM^F@ z{~Av_oRGaU=8*Ow`Xv0bA5z`3iM=@Ew2i-1ya#3N^A)=sm)&y96xm5Tk#^>%P58R} zUy?R1KFF7l4{}lCY}qfS^p-NE-Lsy64Jnj*SGMR{{aa*h+ZI&7`k<7! zVkYit30jh$@FjiQD!wP>qh%M@os@hXy^#E!dBfgm@=*5M4`1*ieTuYI zBj?B$<9X|{R6KNP+Ia4^k9?(1i_4gFV$-C4#2z^F-xpkCHlOal#2(0aT=e^W*dO#! z^<(%aRsFs}e>r(O3=B4#W z{)^YkaB_Qa^j7tK#LKu;AEWh5{A%g?nJU^nGE3@bRGT1gu@~^6`V^6+Ex*g_TV4%c zx^MYjYMc~a`!*FWbJ~CZruEA_%omhCKwG-DK2~T1M0Rd_ivK1+-+ZmlZC6vuab)VY z1$O-<>?3|g<`&vlNxPMP`y{r}@lE#5G)CCNo|v!cJ{!4RC-y~+S&1WZbkFIqY076J zbtB(J#%g?n&6DrKkGuWSCyluKHEn+fMejT^mt9bF)96}!8cshLO3cp;d|T7$l^R=9 z=4i+;U;nvYkK_Ah_LSN*BxG^oEo=yMxcfwjBYp9&{k*na|F7cbjsEBPr_>xN#m{>m zdYJ0x{cXY=f9EIo{({TT+re*{1G)XYPwFtGr}#K8$IttOq|yDH=&sYxwYmJf|Mal* z>&ln;L=k@8=de*uTj*n-y!P`xA%5O}A#SO(lZA}sF5%}rM%+vJd1bEq89okA;^%$D z{Tn~;7xizk7wP=G;uBZt7Y41*c56Rx8k^(THn9);Nt4>n+;bsk?vZY+t$grme3@vY zv~%T`QDYL1_+ad@%f(fm|@=M%<)`R3=4oy*_J9`j^>=fn8FWNc>`_(DJ7 z6@TXw;_v)j(lLfnd3>7hA{X_W^htejiak;ND0Ke|^Wmg;ZaYlh=E$(*XZAcS)!!LR zr(P{)A11g!VBhtDe)!mH;s$v6Ff;CZxBbuj1qrZ-P3X3DQo4Iip&4* zwTv>-;G?)}+PsOcbPC^tAK%%<;t7q6?=IO!ZcL36l6g_`p1+g$)?^{vmSMw<$H#GD6;&R)#7uLwPf+bIDBF| zWZlGG1Ar$#yq7g|kEMq-1LkdV*NVh-hs*lkbJqE%diDCE{4a4chI!sbKD=J3&fFyOxlEneh+Rq2nc>9xjeCvOE&tqKQ2sTP^~=cr zKQAU*bi}a-lOo?#Ur)%ju8{BY-?;sA_;1v@271T6F-qrTT~hS744cilZaK?(27hqm zWA1g%T-z_~tj8Oiwar7h$ z&g}cx7nkHicKvVsf1n4Sp%4GH9!#$XN?)$D4yS3eV$TiEajkq4A2n@3&grgpz7KiL z8vR|JWpdVZXphcWrZ@hT`$MI@qnq)wOvH1}GF{LX!_qI>XZYkCRv&xsPO@$#=bcuI z4Wf^fHKrGR`tA+Fg+{xtv~9q7Y>l!Jlp7V2FDFl!r2M_d=dwK59>IG z$V=k(a#rb5=M`IL8edT33hwIl8NM?$+WszJ9AOyqOf{ZZFwcCrcz))bX9G?x}aV_v96^jnu#F=k#Sc_pL~sb9R%u z_w~U)I^py!(yzoME_L`Zeea!Glcdh0SLr%W(GRE2oj22ztVM6AOVQn=I%iBIb)xDA zTP)*nvHhjay>9TO?sX%~xpO0~pVQs1#kzlKn)9`C2ILPuY0KNu3q>0=IraXMqZ>l6 z=mxS-aYWBWH_(-D={VBYCh5em%QsIyE#*DmN&hJ2B6n$jYTfv1NABzSooH+79u+ws z!987KBcz`n8>US5;>y}_FXxZ?!<@lp@Ap>c9vbRY+TV-D0@eQ1oeV|DkvgHDy{=U9 z>w0(6xyYDT-z_71L?2yjzW70W^5}%(le!)dJ&-o**Y{rNGNQ&O5@pZ^MW3V%N;*|8 zbogdyop9ua90!Q!=)_4k-J%ct@74#AuiA@6{=W&&>N|RUtMi>c_G6CDOC3p_P?p>Y z;ilt7N_-i!1>NtVwLlqe)o3gdu7n7)!#)Yq@EmEsq&HC zU+D6suHR3c51)}SYhwR$>fR0+hsgM1484A8e#)4A*fqY@Z6oUJ6C{n)t@Isn-I=Bx zNLx$S4#Z#n7tkm?$R}wWmeOxIdves66IvAC(BPwfqRyEFu@QtRyG-8G+U4QGgK`$i zDL0;1ZkoIZld+=8%ZrWox@^3BCoj=&C$B^NmhZj8Vy9hWQCI)x*u8k07X1b zNys*Cf8Z^CucU6jjvXR?#Fy@_p72M~+aJR3cFEnLUD?xa=+Jeo=s-WnVIZ$t=fN(Z#;E+rEg3#6k(_7Kn3w5U8q z7v2W_{!eHW`AQqRhwsTc;hIapo5cAL{etYZ5*ipwsJo8nw_i4%eYd%Fb(*v+bFk;H_l{kB6lhnt?c`4Wh{k# zE%VI({TO%u4ZNARED?629X9R`%eTY6=MI}^hYh>K=GtLjcZbck!}hqtZm`3ibcbDI zhlSl?*V$p4++o+-VJ+^kTsv${-ZJi?S$1d%pJG09@|G2@GpzpaH7+Z4{3nm2Yw#~` zICwMBJ$@U+k@}VIg^h+)*g%?aE&G$dH2zgS!@gWAv0eCWB)n&Y#B5f{Qmk3 z;%?S)HPhW(7pNsaS`g#9*%?j#Ac$PiT^?#UZCIZ9hO8-C*yn_Q_dTzDpje?QWp# zO6(ePPO_DC@R3Wp7QN4ve0>szEs-!KJHkd=2&)WQ9>()OMlZPkHvV61Jvv)qsQKeX zZC~(x=8k&ZKU4&*N7zsIk5PMls((-8vNHINebb2Cug{(Le?1ZQqCVr$Kb#1oETtRq zdXoApLZ^PnnOpKvw0|>E@6q!!1JZBZuiuYSvPT}+iO-joBxF@uqV1CQjX3jUrw+6D z|0|?>fOL0wvd_4EAO0fbP=u^JMTa>hl zhp%(bF#gHtdHWtmY~>=K?gNDXp(+pjVs<7{>!C2^w-}`oOT^&wJXkQ{>$M%%EJ~09Gz8d5Bd4vl{5Hp z9W-)-mWObECI981`zsAAgR_Nyj~$GzwEH}n|CI5+hwy!btGpT*pZIBW=)BD7OZh!E z3}$jil(el?i85lV$e*}AKV>k-XC2~hks*EFbB1rJ`Qtlndnxs-`Y!q<(F6HJzi04C zvHgUL?T-=e?>6{=J4v%srHQCI{&M0@{CM9*yi(V^0of$_F8(Wa*=?aO^`q>;PirEJ zDzGO-at@EW=KCV_)#u5|Dm9IOguST8R6g$4_O-vqjJ{-8Hcz1{9flr8pwG)mV= z^PI2m7L`VPeeN`(-!<-WLTb9)_o{R^_2zaXaAFovGrY!QT`L1SL0vq4jj|E7bD!=?nOrK zxbgZSULkft#h0;LXa2SSm%6u)ldGuqes@o2GCc_)?*tN(Nf8a$8I+lDe8?lHX%Ryhy#zf+Nbnt%o(#&K zgtritX)(_I{?@MQ-rboAp7Wghxu5$-e|qoQRjXF5T5Hv+RjaDfjX`eb?GqccmtSS% zhxsNvA3aC-3ZC$=v3U(+bFws^slQE+tqyW+-v2?{H7;8ImFV)=c%{0hQ%8dDqiG*~ zYvVs>TSj_rB|b}FSsPaKOnrJbeqDner*Dfq`Y)Vq+guvUitS2FJYDTFywUZc+(>hZS6QS9gK!42=o{xDt!I;N_d3UL~nt|xsu=?0hQM53wi_;n>) zZr}ljzxqmbqhtFX;GeZQbXDW-sz|1lhApyp7Gp|)&YUbd%q*4dn~4n3*7GL@M#JgS z)xb;e9||7jstz|j{Xz7R_Rtv3TzUu&tOHIOxW5F?G4~)k|7vaHwC|JVd;K!d{6vxA*ht%H#99AAbkUf}zTy;X z8>;u$s<+P7TL;a=H>c9~*aB&9=aCkiiVRa`3**9!rYYG|ojz7R=zCjupM!B<1f$aT zYJ7P)-}t_bG8a&Warl7u!)guHWSn}GM`tr@y;yaQb9IHJtKF(gzSn^cS7%3gosao7 z1M#fj@!i)We&D>)0_T;&8>O`$y(<=C{fE(oQyknWlqEe~8|Bwpht9aTQh6r1Jd<3W zL?lasqp{ccLUoL4l5b`EEbi{pe6YvsIKh=&l$dt9?B-(rYWmlmt7$m#{t5Nb_?WEn zZ28B#x~m<2)YZ&-j=tG<+HLfLPTz61lja}j_sCCeedo?ut-qvU_oz~OW>mQy8qC6H zon}42ofED8G`S?b_H#iY^*QQNoXcI{a+m5wAAdZxG?f^0o!Namhc&A1j5|L#-1$z~ zKP6qFexh6&xoZS}%@@Llhc(*&>_pDmGkbnc{3+hr_MmvkY`JyNbsc-rmnn~I=_23y z{2#aVz;#zU+K*8_$wrL7jZerQ=M(i^#`Q&#G6UeF@jE0RbR8Rl*E7&HeE6M>83)VS z;MzVSG5YlM*rvt{RX(#-uvNq?rkR^U>{YJyW<`NPon2`|h){%P%O|f&q0LB9_DI zUkdN|F^RHi>PW8*bk2v_NM<*DJm9WDc;#P%Lh4_EFI!Lc{$REs`O-t#f_!TSf7R=M z;>9}Z)4S=JwX}a^U~5Wk$X*J+ret06x61Lr&n3eniJeFuWBb?sdhKI&H=X&N*?-vP z)^_Em9sdgZC;0y)^;WK_DE+w66>BN+SPJB~GCrgD9$RyX$E;`#Vz|L9#a;bzam;Vq zddwc~Db2>)rnS^|&Qjl!JdU#mC-!$FC-VN@{tf*Dcdq(CM4QY9ny?2ZkbX)3WZB$NB_b=&(2Obg~+j*X(JlD89*B}d5m2{Avl`nIZV2)>9;Jy4N@w?R4b32oy zC70SdAwD(J0iB1&HI&jOMrpm{IKO{ff9Ihk4J}R&&NjJ)DGslxy;{!LgsJq3}bAH@ji79ada4!*y^qu0RbPyBK<0~0Ip}5cEwlMwnHTx3uf)b!VsQgc zt^RC~{k@aB>6~{c+KXRO%$wIzmu#ENKH9#IXXuB1G+%q>s^a|q0~!Z}qxGB0FqUV) z_h+(IV!Wp^UZ40nGPLnis#9&M?3428+jCXXoS#@gC%RMW+?f5XKGiyF zhPu`wPhG56>&$7b2d+IA{shJ>jf;%c9j>jb&gIU_^CP+YN1hdD@M-krXUKaXIbz#7 zXt0?&{T+11H?-IIhW`AVw`p3@7x*svZo$8ltlgISc+>5fy{^CalD{@MxS%=e>zjV! z(tjd8ke;bM@YsB3J0zF(uO)p1b69^jNajb-@JGPV{4#NZ>8#AT!y8L!$46=Jh^I}8 z(%$aUus<@7IXKuZYu|pJ`Jp(sdR-CU?<4={y|s@;edTpu<{jrXX8hSd+=CL&^O&P! z=G;=AUyx_jtD@!7{sg)t4R7A`UDxLC(l_6a;9oN)YO~HJq-Em)qycNoTJmukj(}t?$4yboQUo4?975;xBc}Hw) z=2_oYy`!n{*AJmj-|qVB?X-PsP{_bznY&%u-6ejy#;s8p?~rbN^A+-cMf}V?iQGve zJ=NKK0=o0^A#~^EPIq43PaoxkhvBkSb<$4iX0CbQ8esmgwq!@@o6!WkWYcHJZ*(KS z=~`&&WMebqe4~9gJ*W66-Cek%f9>hW$zXc!bk1ySae9tt-Q{#NJ(qd=d5y$InoQnx zE%bYU$4PFz%Gyu5Z+|Uw#>`dkY)tV_nzKoUMe?V7 zYA5hs6c3Qk>)x9kABi5Fy-ONGH{X%0(dWRcI~{(D;QRjqtdiea@`=FEJjaI$cwc*X`6YSySlK!^gMZYum{HBQk8{0pJ`d3X03Rj;L zV0Us3>bPjmmN^NUPviH`^!cBi{qkq**tZ9{6!#qWIQ_TkpE;uxJ9yP~L1EQ(=qB!9 zS#`S0e>ylPqd7UW-77rg=gz#{+EVmy3&uLZ_$V+wN*oJ!m449K*0LG&eKqeN0rnb~ zu6eW8UzP58OKpVS2d6vxRCMihzF~c9@pu1Wqi6mpD6IV_XnnEP8mG{AC!U_ZIA|=U zePkMG^Z{uZ@)2Xun68TA@!BbW7k)l}hiWRZppY2{?&LeH_8;eHKaT!6PG`hkT#PXk zuA}+sKl2mEA3q5SuN;onqB473+TNh>d(u(?aTy02tglo?bBpU_Q%-Xih>fnyA@h&*y4VR|EUA%t-n%v{kZ5@m@>@KY}nam4+mD5l1bGys0cej>g z{O0q8^|AGIo)sG^I&Ou=9njop2fb8Y=l;jTxb%>Yv*WC!Zv(od@bHFfK(AR=PAHtK)#u$W;I%&gF(Tx5uKAPwFVjg@n zse+H7dp-T+`~O;SCn9nR?0-3g~xDgQ2f z(>}&rdNr&8SK!?TP9eTA^_S=@8pnU)<$TP}P@kr`hl}-R!%gphc~{iOaXwc&1V?v) zmh|Nn@F4nvKHi#K4xe5b=}Y=kd3De6;f)t@C)Z-`9U5l$TTL@w*Zq^+J-P9Cczt*_ zqnJA9D~$YUCj#?Ccz+5o#oM}%NjeT*HQv{~kyrMAh5a*HOH*CpZ1Jhmm-o--KA7k( zapwFse{_DZ4JS01AHdR_oUA^fVP`9I@(q`l{9xdnD)WQsZc6ms%#y9GXRCLn>OCHO zxpz95%r2mAr4OtONJkEUV}AH&s~WAHtVeK%Y++!jE5B5>lj_4Z_OU+Dv5$1wl6{x* z&o|96!e7PO0bIjF`LUrw;Oeef<%JimPvQ5cG&btqzXx3(N(aQ}?crjF&tl*( zZ{Ip0nrm<)b|gP{A|o?`^&fD$mb)|y`|sS8>moK*^Bu!MdE}p`%pbv_bM@uLHIEi_ z1^UbK4J>qZEu`)6liK$QuXFId^pPI-1%%De5*lfp-LG%?`OwNU+ZpmohJkqas zw7DK##Q3H$&D(qVdH9ErGu`o*pUYiux}#{e%Rd`B#qB-tu($@s#wZB9eciudsn2F#)CWgZl?W;eV3fX^^&*o6jx*X3vc{i z@Q&~ik69YLc?D^4KGt{18;5z9bQkYGerMEf?o%(txA81}A->oOA9Wy?!a;Varw=xX z*>Z7RB);@~)(JfozkYb*W$3d@(Pz;egSIDG=O+4f3+$rb_i+{!JR+SeeQUN7&xUgi z`%ylq^^DSft$3ZX;quVT@5PWEqkHuB0r#h4q}Q~sPy2|X^ZK#-8i-G6)E+PTkvl=7 zdUX$99B#ZVAE7Nh;E_+NJx9Nnua_8=vi;b|l8Vv7)gHsn=9NSEJytruhuwJ?=}Pb@?y1gsSfmhQ}UuqZEB#(VHc`97u6o#>ge zF56OH-oVc zeS*!eKDYZqE!G)bW^#>g)p$(Z)@Qm4`ZB>r?+S+Wtj#YJJ3wBg+51GsC+`dE<{ky= zFk022b6KO}}k{d(i$CyVvI5dDnbUyq91u^((#4 z4RWiHtdCnNTtgY**j5|aP18gp{dOvS%J=x%L~-0ehC=)g8c%&a+-qQS$g99*xG9?_(vS49)qb#?|cu*T3g(lzAEdVLFj=Tr`P&T;4CBz^CLCK`*GZ!Hsy zK)ld)c6Q(VNTwAVx)0r;c&FDD6V1D9mKnUi=HB;F?|j~gML>?A3H;nc|L-I3OQZ>| z(yVUe=~UG%J%NmI&YsQ7u0bwkQ?AlK@UGH7<*$^yTHX^QosT_Y=L0~aRK+@_?q`yZ zIjwJQof2Mf>y%gaPk{cyMY65`cE*Rp8{fmdP01neI*y+ma+_>sc)x7VYIEsa=05Bl z>Mo?Kqkq5V^r)k4h<$_1YEP%>bRYjwwil0>j(jin*1+(_us6~hDHk^pz8+7D)*!>z z;%U(uWOyi^7Og>sFUQkdEY^YHS_d$@Bzz_)ga=q}@cTwo_i0Uq@E4>HoY$CkKCk>Q z=B?9iOeBA|!@ZmG?9)qMb7{~2dUS`Z8;oM)MImfzFGbUSqE@565Lwg_NiU82bysY;#;!6b@FOw@5JEj z@;t0GhmT+$fyV|AE+fKQVYOp%-M29Rc(Hqrl{w<-dPum_|d}TbWqD>+fyH zeMhO2Z&v4hZ%ofm$LOgt|ExZJ?Yzbo-w(aeQ)A5Sv6#cPYy0K@kldt@i?}aDar)uQ zly~#xkaG0L*goL+@#|{02JZXR$VMSL7SyG4JJ7?H4a0 z)rXX~hi8w&WLNhIr291MyZ6QVPW(db zNFVK)C0{pvFPpSDFHnEDZ>(>x^KgjyDWiw#_TQ|IR4Uu%XeRz_I~sp#4O#ID!5H)B z3D$14i*n(($j4`Tvy!h}xpX2bS62UrQ(4XJ1xvpn%a4M0{r)b|Vvst}!8T^KIRAE8 zJ$IzCp3b(<1-s1mh0nL8X48_ps!F)@e1VwQVc1QK6F$#>xbK$>*Y5xWIHBuPpU3Fh znaXbQbR}PceE$304#q+cgSN3RuF&%gzL1sG+525N?Vb5(hVi!9@mDkX9-xg6l>7@l z_3Q__k1>_BZ!7OSmvq@D#nb(BT#hRB&dpvv z;lZ?vZ76Hoj#PG@Yn$M4hU0|rNyA(71*1zK+jn;=TR#um85tto_}l8ObMuCQVT~dE zqqDPmD8Cq(+E*o*;aTYM?qS)!xzOY;elFi$&SOE}Y%}}3V^|h{bm5nzvF?$xa@VpJ zG%2bpPx*y{Imf}A6Y-Yj>wT@jehs*Xf z=szcWwrhJxx?*zMiE-P9PPVl=?BH;AP#BTGCa9%8blEoidOmhf=XGsKYVB}Ee|>YQ z&*T#@zMtvpoQY4NKDnoh*kZ*IE3QtlL0!ZlcM^x()p`m47F*0*c4>cntuow98#dF1 z)x)y;nLF&Cl;C_G{pNG_AOEB0;}1QK^7;|G#kzQx`GHtt!H*Hm_gNQYZqm>Z#iNV1 zdEk8j-t~K1dg@9%{e8}9^m%^uJNg6KJo>(YJ3Zg*s@}BZ%PvpY9HVE;x!KUkgVH;b zW5D5}{?Ch^&gY=@v@&|KUetwOq!T|@w5E;km$}N53u5$~;^;Za(Q^{?>=iwUYbNHS zi+BXZ1a!43E@T}4TK`|tuRiZ=y{KQlZR4l$(D8ccc|HH#$%vjKp<#`F&@;(@QuHKF zoA`mw0M%v~4qIIO_w#MiQyyHaA(NSIU1$%n278B8vJf9Rjq0K)+ z&$Rql$c^DP@Xp3IH$kXZ-rB*OLHT>I^*zpG96aqcjHi#N zs3(FM*(}`2HAyy~#<>xU=P77J+|@dK3pr$VSrI4N4vh}lUCa6=+=w%v&n@=P#>pAz z0k=mZkPd6F&AxgLI*f5EWPYSF`6uV+23pJaI6T{1I(Iy*iRkR-q-FP`G3S<#e;<9W z`S6ce=Tz)?12#pIix;Qfr-J-(k-mA;r0fFvF-h9$D<@?ycm3h-MwCB__U@J6;X>Ee z%JSFv^2?6Pp6|<34!N^?V&}bpP6cOZbAY}#dR4;sgoknOq%6S!cR1SFIdHRXM#lk9 zF@0s{^CT71Qqy&~yL9LDk+Cc~&u(S^G~s|=@9dtsqm8!8r`CRFu;(S#!Z(Q?%%`-U zZ)e9;&Y79IL-os7-qnpy3)`%?R|(zPMS7**P<}Y^1^h7XwHQpzzc=ViGJUtTx z&W;rg(1ZA=(jNLw>h|?&o*;kyv)H7TcSIz6=1asMnR0fy$6Ye6x_a;z>27STH!0pD zadK1PTKC?APw66_&!%m@oQ*jjJ|}wv>G($@o;|ZE=i6O1JnApO!tY$}FGcrxIFfhO z&z#VF#Q6Dg3EJ>a?%nEn>YVJ?^^UIewE8z_C7-f=f9{;@-F!D+7_@40F$;QEJBoF| zSMLxUcuIBpyG)H|s8_UOpL5QiJ)m^TZdV`QM!8hL`nA&&K3zVt+xZsH`zH1mEPKD! zuoiX=B&!DXJL!yd`ON#kDP7IHPGd6j3CRooTH|%Y6ZmP)Br+VqyNf#&N05IJc}G?U zh4b+X{O5Ic2e-*4^~nzjPn(SOVvF@8u>+mY_c?qbWpG-{ z*WP+m4m?!mv8qjbe`)#S=iq9cEIKDU6WfnfTw`P}n%2Zre+zxU7@C8hpGxf-VP^=E zwpa3R?FG;M>{&aXe2Z$In7r=ewNp5+V%`4a2x1Y4=Oz|&Ke#}Pg9j+HZBej?@0(ko z1#-i>_7js?!-;s4_qM|WHXqD`dmrUG$*;YrdRDzv(EeW9k;%R`4X+)6Vl1G2Z}lwc#_E;#;Xl&BE z@(DIHI-qpVJDyKUYnmkVq~Y+`l}`SO0ALA}EH zGdf$Fx)sYWTzY8BYwwHj5e=fVYfJm=2AWH}Dw;FqVqFWQ*>ztJ!`p>gyD#I%%!62*09t(mDEiq+BiV(M>bruOa*rbq47`wZI4 zH_=diIu+h1*Ab<-0c(34ez0W~w^ETd;#d(``k)8pjbhvUP=LJ+$SSNd7|h4<06cJ3u6j<^_yr1q-rV(bskhC6ka%W7$S(Y>VNg%rLU zjT^Jfzf-#|Kl8}Dh}9oOyd17ZWBdq%rpIaDT5dxK);C{?<`>gx13YfyvdQr)I%A)H zXMRzYwE4w$%`fIdGquOJGf%tD>>K*t25rgndx zy{x4%Q@lPLyNbC3V};H;-`YC0zZYLX_cakO8J@sub27FQUP~KKUB%eqbiH&hy0VS7 z+B*F`LH1Yd-&Wa8q{-&oSMd&Ao|!Az0yPaprjmpoL-s1Ue{(r(x`Tfu4 zHMQt4aQPiFWi)s(+6!*^v-^O!0kn_@5E$IkH3!%);Go9CkE?R#ote0?J)j6HCW#sf1eSoe>DDn zGWGL57ksR}J+9Buw~8sUw7yc>TYTD{Qkr5IEZ@&cX(#!#Kb6vSu7l!puJcfxoq-x?3~w8jeWVc{fEOf#qJJttCh1@C z5OWXHp)X$@jgN5{>YL6(D;t0_F5a)mt=ZFpDIGm)>?X!Z^fdBd@=h#ir(5HYE>yjW zXLWkf-UU04%oKG%YYnmFClX6Om9uqgviS|@8{H5Ee{y8@ekq~ zG+D26_24UzPf+Dv0|s+L#Q{0}QkC7s-BW6d*9$*#@4~^Yi<{nAkG_cKGdbE`oqf^a zrToDR$^d%?XJx({Sz=5y9Uea3Wb*NZ_iw3v@_ivQGkluMV>Hosor4c8R43>Bn2d%U zj7iFe59P7adZXcI`1B9YZgF@juY3l+T`LE}|CpLAIGOGc{(Y43z8HvqIQhWxn2R_}jbWZ^zdzU%dSA zUHEn7&(&W4F8sSCUo>%)t@=&DcfP#;ceh6Ut2XJE2j*A6-N!)o;G_0!lb_4l4$iba z>)UgZ>r0={Xuz86ha3$&+`s;^yu8-AzYFccHO#rkxp*#UlrF8S8SOp1D`W7=>iSqs z_6GMY53J{P#|-1C;mWwE`2B%-?tLCKZ;txmq*2bdLYeGK;D(QLfU>~$Hpuqi-)bTo zgtH&EzSG$toc%D}*~in}nHXw=&cx^gzV=OoX9a~&{Y!gvHlN>fuCwzL=Oo*I_tnrj z?!%vp?wDJm_ic{eWo>2McCv5lncP(%zG>#30M<&{Su4>xNwi*~cstf~u-D~xgQgMR z%!VGT6A|Con$39fEQNRYXN{)qqQ>lF|thummq!`>StiN1T$v40} z2D~(ui|%fn!1{01VC{fELhL=`4a$DR^M>dLZ-w;Nzr^HfKJ@zDU~nd+vOjll1S^BQ zR@gm;!vgjNRN6hORz>v>peGHF>tzqFAin;}u7TDX^VN;}UAfO39V@wCZ+2jF5c%_( zBYT}O$?PfFHYx5F+^zp%K`zyVFK-z9fZdWx!Hd3JYq>4#%lDA4YJ|VHIOb1_*i(Ux(Ib!?kQF?ZgkZ`vELxs~Xk@h$+C z<~jR-w;h{FX@Xr#x_G&^I-sAtuQ5LVD)~`7j)ot!NrTrZ3B&Dxd;+=)uZ4b1aps$R z^TJua3$=s!hv`bzM@o3pM)_l$Eq-9D^LyD|1io1xGSB~K@YVb&tTA2kts-CS;_M#a znm)OqDw`qSWojGr)|j5)oAx<;*rh4&H9Yt6-N8aPK?mv(uIXBE)jM>tcW6@PZ*hE~ z`IOo|i*}Xy_uO~E4#qa+hT;q1raA2*`X>)AZ?qpmc`y4r^2Yh?B={|Zuli`PBgH^$tI8S{!wJ*8vM15m9U#jd?=kHpMvPZSb(mardZ-z zQ{P}O9Q-C7DgRIxGNrkL_*HX>bRwE#x8H^CFuKS592NNZKAQYhpEriP>t|2Dq57FQ zeNJ%jRpy4ncYkmDPl8?rd@_G;{%gqp;(sFlKJs(tS2_IdlD~Mc`89KoBj?w#GSbEJ zVR+xH_Z1)Em#v_)=aJ;T3VdR^QTk1DYt7p@gVpAc+Rvmp#J0~5+__Kk?B);XbNn7r zKW}9GDw{71)o-!7RELLc{xtYZ^=tei*4g$s*?J0k1z)uJ!=pCFbmk!N_JNb|70(Km z^in|;yCDu1uZ;<|!f1tgd`SEk|#mOfS^Owi+ z`@W=|#r_nH%i9|(8O~f9ZH_)0m=oM+V-jucoAc<6tbycGV|j)q zBa+d1O|seT&L-q0l?c}Tqo0pcX0W^;?isifdTiso@9YBTV9(^;iXLE1t#ON58BhkJ^FrkNF zl+nI9vNuc?xHG$`PX>V{zn<#U_?-~11FMRE;XfEmXn~$&e(P(-QrI0D)9I_v6k)f<>>uSNUz)z1HL=vU`TFRdP5e1Wy=T+cb3`3+*`g1;%_Ls? zh-^a9m;SqsXS0RCH;vs^8`K_R;tL8Y z{SU6+jK-cWLChYE%Z0;L^)8|=zt>)JueF04s8jN9x`n=aGyQ4)U~p?+6YRnFOaI3B zECyRTq}|~ndTgaGjfrm(45t^R)8vo!dQv(PpOD#sMknf#EPVYPf3q`c)#f~M8?UqZ z8Eof4_{a04={ERMK5W^ZqUB&b>2-dQZyfHHULE7r)^dNg`bm6`EpB`-8@%YZiQ%<{ z_DhEeFO5gSOE_(T&+-2pi5KDaP?MXRN(P_X4-bOx+^Zb_;hR<-<3D1CBL0i%UGR?Yo%MXU7dR%P+#{K1 zpMTl7Fhm*6QMfl_$T1||2FaNE0ba5(<*l?w<&AftLBy+ZJY~DNdqm++f6j~_AM9L{ zBy(Eec>e6`VD3#H%8xz<+rZMPvj>{X#ygN7xD(n%el_)#`~vis##Yv+Mc=)$8Ithv zPX0@}dg%Sho_>BzU2XP5%KU!#jO4>_Kn z_JSTSn`dQz?HJyf%hfVArba#S70I>e^@8e2y1N8b4n3)85g zEZ=_U<rdY<~@9|mZ=pkQ+wVD41T$_KuINV2@B}3_x6OCo- zv+?l(8cXjDW_wKnmhlvP>us-c`zB`3#roar2+3)CeXwU4w#N~D9m_uypOn`XPQFQAO7a;S<6?A->mSknPx1*$ zzk}P5W6a-aO9`IF2hDK?gJpUFK6xYE?rkW|eTKqIu)J;a|B4OOT0Umlc*%IBI%P{q zmuO6=9J{QI0M35`uLA#C^>!>3dYhpUu!ErDOsZXB& zBgwbw>-utbZ`Za*UhVtK&t9Fif8f;=d+qgYu)J$O^RIimF6wyoiFv+<_2_l{{u-;~w~y?xPU~p`jB-T)?T17G(WL{*wzVoVD^Ad(zP!faC1o7`8Z>j=xupu z>zC-iN8LUrrHL+&3BSSUa^~Q4=@xCzY_hrXyS;zd;jquaisSHt=CS;2|HmM-yBpfI zAB8^f&1Uga2>$RwPWugX?m*8>#&?Y!@KFbA3iqh4t9*4c5{;sFDz^TQ zk86U#IC*|!#@}tByx48Q{H8}z)2C0{qR0ST^$Hf?en7!2+o10*VXnfn6tlQCnoqaF%_3zHpe00`EahyxW_Adw@Y}l>6({2Et z9m3~p=$)?{4>NaQ?2`UTfk&k0-81;f=ccuuJmZCA;#`Nu$<&)=4yO8&dR})|B(jO$DtzYVW02JXa9$DK8oH9&)C z?s{fZ;bCmqFd3X8KF~V1Vrz7k#)ZH$8AFe)Adj8}X9;l&dB%0&r8%8+u|30+I-jLy zF5{DELSM}Bb~0(JH6G#56707NHvQ3yU+X#PYg7oiv#m!BBS7Cwdq0h3P zlf2N<{CHaXiReFV$2&-e2Q9`>yp*DDPT+nzzCIv-#NRI0`9v{Y3x4K90_Q9=YxjFi zp9|)+ch`OI81)r&432L)oC@B5caW&R8~yyqr^rh>VI%c-2glEtro5slcg%lpDE%Ct z<4DI-ZyC&mRnWX?`ZU26%*)QgS4Z7!iMYFtZ$rVA{ENQQPdX0>{@v3vFUbCy=b_37 z7wNrO@I+U0-Hwcl;ZPd1J1E^F`7oKDKm9<+oehd_R~vPn4Ybmsf|Iv2`}BLzPFT95H{Oz8-k!ca#Z&U`d&uthuD0g=0xS zm-DmN&%}u4o`cSQtPmYUn=bUMU*k~RWZXu*1UV*;*=?VzSmUV7+xT<%sbU?YGS3$3 z+BXF|ioU)L*m2&re!|wPETg+edw?xD^@)TKJ~{HNISYdqV0jP&X7YeGMCO)>3e-uczU z)3%Vt@0m0IX5+1RYx*wI`29Sd)YVl1|jv2`x7K%#E|aAlB8^L^Kv8I45PYaDkup602n6 z3eVbirWln3u|t;5{xi)N()cUl>m_A#0nyyiK{`kElM>iL@#;pc zj{=`Gp20!SMf<6{G+qrRYsBT2mM=Pi#WYO znqOBD{Z)tHs~*~AvH9vxJ;&hQ;*Ouj|fLSM`;0RJX>1uNF2wC zK5X=N!V>@1=xEs(wUxV!wU=>TQ={maV!YIQ5!M^@DSm{Ku5*3*SUhb${mk!?LDG)s z`yuL*ZayNf`qk->BCZuSUS~5tBKlP_=Vk4(XG-$pZ6wJ++@5(;V`R^yuxEN0kK`-Z z{W@pg%APYCF(v_1W6`0C@y(A%rU%~D^hi-}#c<=?gl!>vl(v9h7jX8uI@BJs@sLmE zk2VJ8kXD*HimRME524vN-w zN_LOz4f*oEB)=5+s4pA;64@1Td&ApLmE~3T&&=JXaU7wfu zD>e=eRR*4x99tRV>A}l*nf7uZ`{2K=jN>ss{)OQ2_t*jA%OdXJ^LN=t|9@z2Qs>`} zW0>#n%b&@A!K%E=_sl+sqsr#p9RX`*AG$yyUyK`!yOnyC?6k=W67|?)-Ik8az0ewFL59 zjZU7;9H*Om4LOT>t@F*vH`}E&WHZIwP|vfphMk!88Ra*bQl9$69_B)*(d>p4aKiTkL>Tne5h?~6F6!kv%WT4Cc|0?tnNM>a^sen=;`#j!Ld$2MNf zXHBGe@O47@0KKnAGCX~9G>(JtOzC zk>$@KgZh~cs6EEy+J8^oC>;>VW--m~ji^Yo`@|~KO7btdN&ffOmFkYa`{98Wc zUv!yApId*@A8NDag=O(3t@78`;t zZS`@Uoz`WghqkN^_I&w|g2~=H`Z`}9rBBBnZuQXjJ-z5D{ONDjTAw?MG*A97QBL(p zPs#Usa4GzPj7hdUPn$k^Ze;fS+02#MPbT|D>mliCWRbSoGxn>>=*<1}Sf2S8e|UQM zI4;(QA|ImG2goyc%+~q%m5)4YCvLMymR~j95Y0Kt^-Q^~Y`QDKx%c={%534*@_gQ2 z(R}@7V3+5O^*eo{F+zK%H9ztF5ydUhPsDFtPCps{Kp*)CH8)#E+eKTY8-3x267PsN zw6+W{)yhte{O%toK1Vvn%D|s?XC-k2S_jjZE7|NR?xSHn$b7Zv1}C$ULHTsi86!Fm zO+kk(Z)kJpxmjKGWeC3*{Cf7eySrRMo{?+$KIlW@3iJ*Qd%EExc;!;z1+Q5-_)BJA8zr>eL!tp7NK%Tb7^o9M$<8_OjlEQ`-4D4JYdDO9ole3dVbF$7HviqW&oH z#aEwVes+}ptDaHOf63T78*-_8dG^r`-;WwsjE})Nok%pQ&02R_A$frQ>aRM|r5jSm zxV?^~1-pXmI`u2^E&V|}fbDsf?`oIx@#ax zKaBSG>$~ss929sVoXIpUiUoDqS(hgRONDB8|vPw~L3n#)GC-0jYY5MAkutau!M z3idehI58QE#)*~vvLF1IApNfMI{!r4CU;KE{pbSiv6GGvZDu16r&I^*MRRg-$FbSx zX6cM8^oHuhKJa5ug0nC`4bLeKNjmuIk(-u$Qty&ecnuygd&OcS$fG_o`E}=^C{LH< z85^XyPmeU|gQ!i!P4Fxlqhrv6#K-WgpUToV27eoEurUJsPXYfl_?t}MIy(C;a2Iah zGCM?gK_|19xo@yY%Sfk4K2RM(v zDgEYXPNj4!Py7manmsopo^KkP-7Ff@rfuLSoyPvj1N3K(eew}((rd-r`ldS2y@BI# z(IR#5*;3sxy(}1p2eB}3h{uF+*|p$d{X%`}7tOUa2MOS{pZ^MdJQLc^)Yx%V|31=X zi-x@S@hlqYTseI|yzv6=P;ldGYsDB?Q`*CJ9_8a>VA;I~%?-rRbLMxq#fBB?>%PZ4IMCq4Srq@R4ODoeyMd^zx)1!O)!bO$o ziwE7QkD>CKht&ZeDLnLZ)Pe^zDsxG4S1VtOHLg(hb%)H}l_I=v&VF zrMxfa{nF+_8ktl1zKshCVefd`H@^SJRfWFC^;-Z8{=50_pxizB0i%Q8FL-`BS;#*P zoL}&JGFj+(vbpfsrpT@_oI1c8nD4H3KGfFn+4u51aBUUw6yvj3@tn~r+^+Egzty9qv=-Sp5 z{e2zdvV9N0|Cd&wd#>URvvG~8vuED;?1w0izQp&N*w*s^G;SvEBIv)ks&GBu!g)&@ zTZTt+kKEmw`n?;2pt1CxT0?Z}2GJ{Tv6}al@Y8DESC;rG-!V2jaAs9u@qN(atOeXT18%)zvz(`)xQ$z9a8BSw z)U9*L^YF#Z!h`2ymkiu_v)bwE?^{isuY<#D)cHCvUZc)+)VYQ_*HP!1Se?z(IlHQG zy6T)mowLc`N}cz(Iyp~muIjues`GuSljnmMGY|6hC8_Ta^}S4ehbaFt^=+WOYpHJo z^<5jQuXjv#psuQLoa(EmzB=;PQ{SDgKF+qAL>=Ls(6J6#scSA&As_jA;ME%s*tz1? zcXz=jDc47W(M+CZm*;lJPrc(gH-YqdF8vEG{R`|-za1GUA49hXnlAG=GTo*=x#1Na#Mv)0A|lEL7`sszlF1)ACJE0Y4_*p3(>=90$+qv zfH$SN@DMQaGl4(TwY}A~sWnD7>1Vq1jV^s7bh~MAy8Q;3&R_5H zT~D6X(5)A`Jxkkrq06(hzZIA_LATas{$q3-&G%MEw+G4pfPUn^pa1*$e~@wy=m*Un z>Q6zoTv1LaBo zgwx-i?_|S;&_VCQ^Hb0pd(G^?BXIfaaoLgJ(n#JATJ0Vl(apxP*GFep{(g|LtS6Cu z8)xF>*SI#Xfxo&(XZuzLocW(%ER4pz9jt4GTI*cdgdcfyHhe8n_`YHpACJZZoxkM$ zdBRKg{;Ds6bD9cY)O`fY9sK3+`K5#K`SHhRfAeN&Pu8o0Pd!}*4uP9! z?)8+O@2TQPUv0tPr|*a6n_br#+-mWRMoa7~!6<#}EPcbqvTuU*a#TJ%kgz(#1Hkh9 z{5;?EUdhic&G`9w<%gfiYcP55t1%uw(BkTlQl;mhQ|8Gci}Jo`CCo@e&F-LcB~ z+cnSC_wX*-dY5af!%^+q{g)c!4d2FE()#WRa%mSM=i41$Lt#wHS^j&P#INJRI^+&q zYmsH-HOtt4kTZ?HlH$He;hRVP-|{kjv?%JM{385YiUC{b`fMTXcs9WqgXn^15``4! zeT2VA6go%p|8xT1V%46i(VO-%4`1>Q#)PnY8e@5CTi=u19q=r4+Q~olqtkK&wc^3h9VBnS1%~jd({))UaOL^Ob!&we)_AS^H6?vae74ZmH zmTc;j@R>~=(pT?-UUyZLyRTGE_=ERTshhdqh;HG052R8D%p5KOJ8pEZ_8q*^q zSvOj5CeJO7?*#vRb8yt}c@$t3r@nT!u?gzne#dmcJ0V%&1y z!-dNlTl~0n6uGjtP_D&~cjOI$X=5XDDB0`j<-Bt7@Y^PHc3q;7se+HXsc$2=uW{{p zx+*ttE#;p7M6R})xMlW|bs}?$PpIPkQpOl;Ewkydy>cP_lyU1ap?LrPD#^)HsjH96 zhRcCdCRf9Y{G1W*-{ta-1)rrAdDoQk8lCBTYopqQY=`vyZa0>tYk{@7Dwok5=jIc+ zC#(Y2%n~fYFgT*Ci+2}|LTFz`tJd0xR_noSuA|jD+I#(c>@r8IjnJx@azoJSm_caO z;%L>^-Ng6<-b-WjniJt&4|xV05y!s=u%$zTxClb*DtL zxm7v!EqD2r^KNYf&&oEJ^p2Gi&h=IDRXj<89~|c(2(?S zbVWHQFIFc!wcE?e=4Noe?wjG}q{%C`U2dQzD(m%izBQ^dTnw(hoOIWP(0E?VpT@J` z=jSyQJ|UZZAYh)3O>!-Bp59T}({xtyS#BIUi?Xi=c6U*#Dze}5t8237&v(9kbe)}% zN7}Ll%)`$lKl2uJiTqUZcib9$Z*unAZ$e(EBkZW+N13JAcT10A-;pORTfzCbX-{O| zkv`%*QQc~Twd3`s!tJb2JipV~X1s@YQqP^VU3Sa?=5NqJcUn~)%eh&L!IS57HSmdp z7_l}rE{z>>!3y;34AvK`gIqWdxo~T%Vcp`!(CMDCa*2tjA5l*B(y5FUf`6)~A^qWD z@jc_upjKXR2ovpQKb6CtFgWbf9PHB^Y>oBSmZ*>Hc@)p2)s1!k6ZuCcW(IA;aYxi+ zyoX*GJpcH?^P@Yh?zPYk-C)lj6>ujhbMddbvZ3aQlbCN!YA#Is7&g>w%FcFWXFEF01_w`f z)l*MBDmRC`b6nn;uAZ6HQ}61*$MA_5jy4aNSCd`AnH5GS=qw+$xAE0p`L_N^a#iQ* zoKE{7|JGhTl%#Fo)7x#9{u4CS?Ol9Xn2gPv? zG&f}BC;#iUk69eP#arNeYzrcrP%(BNdV>4V`QP{M^8H4G#cy9a7PvJP@oyD=dVS9) zS-(`?r1B22&$I^GxV#@xOwJK`OEEdfWU-xE7u34vU@}b|Lt0EA?d+iq`@mg$U*E1aLrcX7m96`;W_~!f_pBI?Oh0N3Erb?rtm}us!p~uQ z;8KZ5o@^a{MpGkg$yuIBO}SE@z+$!ko|sL=KCJ*hvpGhWpZD`TvaIiC!UHK6-v^J` z_)QF$){HpsIu@^q4dP=>6xU%os+awp=n<`5_n;SIcaR&uiZ8WR&6taSE?{q^TbHMu zYQMfKeyvBegnr+`KM=N(FL`3W){k@_x7LjOU4n`?lE0lkjBEt(*ZQ$&ICZOZK5#kYm1=?ahes)L=6As{zVDmooZS@RQ=(J}P}oUl^UKQ)}~z5&DqQ9eu@L z(jD}vd}FW7x|%cg(HqfTmQsxPI?*Iv2e7=n3FdNO3RjDXklg}&lcj@V*>?beK{Z75%c2 z>|R0k<0}sCr9Z{u;5HmjJnCf)8RrKRjAIwEF4@ZY4%jYj)sbIF>02m&-x%o%?kwcK z!ZvrWfM}<8f0v>9Q0E_aa`tiB#h>WxZ1tzo?7RhHAY;6qOI7#)qc{lSN@F%+XE6?9 zdH>DesCozcdE8qPk>ASW{(0f<)}fX57<7$rzZ_h2Uh{kVmlMO;&iUOsvpID!`lD*P z*=`;t_?Yj=y-RL@LpZ;?cucrG+1utCAZ!x-XaZ_O- z^%1~_=^`!QR#XAXBRQRjU6M`fgU zbp}P7pK~j%q_g*GjelUK(>Jptedp`h;^K&WJ<{8u<5$H6m~J2~T^$rS|H{tPP+F(A zyw83htgG_i;1_hbQtT%JGg#H4xwL2U+Z*(@KZe0SE0wx z@o`C4BAX`H(jkwMt~eGwf0yT|J?N8GbZ@K9Xs0h2bBDx%Jn2Do+!g&T4u``VFUJCc5{%Yx;r%HPe6z@C%r(d`<4 zkkxQ0G@D2tBXjob=FF+YnCr}cxU#CTz4g8Qx4|F&EC%sHxRCtFXz12_3lA0=NQMHZi(o#lXDd~r(r_4azXLV%RIPoeu3RV)p-bgG}Fll{H{9~Y`iJPRv+L0 ze^Z{blO}MUP@&WLEu`OC+_P`X?!@kB9SB@gqu^`)lU7f1o9eUpJY+!q79t0_qk(hk zCRji6UAkr9EPk9FWaC+)|JzX?U)rD2njAVtawL8X$#gIJ69N>u=CC?ape6 z#^uNk@H{yfFKW%L3tJ)1i@W`JOZwLwZwp8NTPHlwvxYZ%RkUL+HNo^3^s;nl6X2KK z!uT)zYqd_onnxA>*1r|U-%I=Z$OjJ@Px*f8Ax-ycoCGa9pq=&Acvs(eXg7!aXOYL3 z-8sML7nASn0mpBK2Q=|(ec>E@bjgTTMjvb^jdfmc3`rmI3`!r;S5(kPJg)nBRF}WM z!P`AU!cPtXpY(TCz>oU%_bn9d0P?|rXXV0)QdhV{B|6eZ>THYI!g{y)oo{Omd1~NqAB#=8 zY+=VXYTsa+){Ngw|Z(8_G{l_>R1db-TFM^JX;DM2mJj-r08}YZ!55{`{9Q@K*_SX;(6v25)aGV~Nec|DR-rfSH z>|oPnn}ggU(E+-ob|q{bHU)i=sdZXkBW^vXZ+^UMf1Q|Q_+%IRbsl5vTkxT`WuHTS zw5BV4S#`{&y)&@YXEaUEK0uE;l7e`YKym>;`CStymgMQT#hcfg1v6hJujMj zE8efo``l_+1NZ^Di2rK%J$}ZIpa1)FuH=)EPekMV58&f6A3$qGd~6bWkyrkc2~*1B zW8-sqcs^^g-b`F0ZR!C(`7-3kFx>?VFUwW*%Zn!oR>?Oo(5*eA#}ap*%yxALd*BuG z-`Ls;eJ*~IuPSA0!K~BXlxUm|{+|c`5ZwBJzrQ(=-Nsm@wVV4HbM`a#-Os+!ZP>f_ zGoRju-TDD!W&do>eoNMF8_DxF<{sOK``*U9WE=C6qP)UGBWn3MxeO$i*nwU)&;BU= zC%WJ_(|jdnI~2ltaIdFt;c>~W#vAk!>vhm_Ze+)O`9yqyqfI8Z<;CB<#2g@Ry01t+ z3DIZ?d5m9xrTEM@6PKfR@ok(A;Z5@!+n7_NB!lwvftUH@bsoT+Ajdq#=Eq9+`~MGb zd=D|v(Y{IT*SG;XR_-rR{yhDV)>xGk55|11G5wz|(~%`VYUk=;k9ce+<3k^9)tLlp zYXW;`U(Nh){!Q`(uc%GplQ^9Cc>FK2qf7=$`g$<_f4V#3|94qi%FkSj!D6dMeGhII=XJ4Z$HB-tcI+eh<`Gx>fdx)lD?JVNs% z*?I80bn1cp;!-Sw?z!sW+lXG;PP*_#_Z9a*h4-+B=brhEDYxFNyS#K>=?M05l#LV8 zo!Eq(MOz#@iFb`dkE@^NIoR{uTv2DY16%c3ovOpl=|xA`o}Gs-Z%8HT#`VqNEJ1ul zv#KqQ8$&9CKG$wlV3f1I%~)6$$vfD`r`Ks%=Ymxh#MK# zmmINe*3_okM(Ex$(WyEbTm1Z4vSQ=PZ>D9x_^wE1q(4$lcc|~)4&Hh2x~~^HOmTgO z9O&*jc;$$`KE<_DZEA7rQ1T%nZx31f=}-FC=2D)<|68%sG5cTRljVi3DZdt{XZaBm zH$-!0Xs7<%M|v-PbuV>%4V+FwhLLaXNRIZG2~K63l-9Xnc>{VT=j_|3Gr(M#S zGrj-4{9HTJ;qh?=xjp(9hF#I0@y|*84SdsF!tg~W*P;(}Z<}lw@y%{O7Sli4>tj6c{C(=PEu_6?-{Cu% zOBv0YY4=O=J^UT8kpt_mc67dcQ!SN!gdCd;7v+HcI>?IBBn!hN3((j45gKOP_)Gsx zDDLkQ9<%7zqHi~Pm;QI<3Tks7@789E(Tw_Rl@n zn4Jz^i@ud~QCjDrs~W(CHG$*z9_J(Jg;uP{-38PiL~C$c9PKjAs^ zD^38u-q9bdCGpSL(Bta1bmYnBy_vj*tMbps7ICoC(u;?qc+Y@6{@lzzSM8_`cI*|Ns~w)) z6To?jSM9()8tZhH$yOh<5N>Jqvo9e}5A`-d`{2=OoB_vP70yEaL$mAxY%0+W``~nR zz1amdCI5xy*WSPE?N8y88gxB{ea8A$Ifq17oa)Ar=c&`^k(!>BJ#6sfbZR*%`>@i% zH|}5UY@SN{R}9hq6|ViZ?ilG7;f+3(ul$YYzC++m*#ReO>ihWmB=J-5eGPNn^1Hz0 zZz${cf2hyiLVGl)-yohuR|uBIgFk95&Ba2*dD^dAiKhk6@1b6$m~8sX_>lIA_r-H# z#B((+oxaoBgywr?c))-E13cs_wwRFW0Q#~oihODD8+wO+jp9XWqw?|}>YMSEZQe+*--lC$O+54Uzt-qC@LdawL%jFTznlcP=BQ`o5gC;TNmig4hwt-QAg zF7~PU<9ZW&c48Cj9GNbBAcIP%`aqpV-E2H#ek!C?b}T0b?%6kPHc|yOk+<-s5uRN+0%=ToEH?{ zL>h78IV-b}y`(N~IZxR~1TU^P(T!$rV@vt92Jx4bg?HL~Os&2%FA%@NM|(AQvpn!Y zfNh$evgsW8#pF+wFM0qQUElpr^Sb-R-%dWH8)eIca|9PTMu((+J0D(*&IX$A#txOU zvviy-BOkEj9>0abZf~yI^FjGA7-t>Mxi%LQypw!qkWV&|<|wjHej}YY-sK%{`)top zIn^@Bo!zT*dj}tXRyzG_b5NtRjf;N1Ft@Hj<5ZblL7MRGoa^sx zFU7H(R*5e**Y|t^Ej?(=yW{Tf==$FZo%lHUbSHs zdIcHJ}uWC=hHb955GvB zm`;eVHO1TYQ_4w>Cg;hIYkyq6#yPh)*O@8%m z`LRFt5xZ~`<3@Boiq0fx$|;^CY(M!fcf3XJ@~30(+8U zbT+X@N+F&4;2$=a#Ol-A8_XhWpq zqIKVx&OO2=i^kUS^Eh@F$JVR*cN1@|y8KVP_b2q9-q-R=2?s~(IBnChG1<3gw~Ij* zFTw96nx016=yUaz*PWe?p3-L;m%-^F)!D_fcsR|u10E;wbxM3a5gumVSjN}LStvgW z>B_&K`D7n;rSX-#g)$eQfAmY~+^4b8xliJ`3-~4<$^m4;=U3zM;aVPhFQ+0Ez;*;`>VV%%f1fm$DC_>ZnwL?QtQe1C``^RKkv%FN%>>889Z}_%BrnT z4}upqn9)PN)lA&p;@#8mTi)Ih4^`M(+|gX3CvBUdw#Dqdc-v%~NlwC=fPN$X(CH~X zhe>y?4ei#m;fzhuljII5>g{N~9DCsM{y(Y>WwzOSvCS@F?I5zv{`^RVZ8nm=^R^E3 zNhP4IkGlZ2)*5B&d_ZylUPrTaP6YpuGJW7R0$ArKu%F=7l{ARDQxd+Ltej^8FS{HD0>Bi56#Pda+1b>VxOfW$QQcUK#SjE%&NsVzl}QL z^cXSmp2;cj;0&R4#Kz@jB%?m^Wl|p>wiKMM>(kkeDn%C#F=SF~%oOv^0}j z@}^W?YbN-k=nvy$fJ&dlB^+^;j9d)?)0cJf^0Yvk7W8knqg7x{V#vFE}|Kh5LfaTxo) zcceb)L5{pm36X;$U9f>xIl9RIXMCD^QMAxAEWM8~f$F5=_C4^T8_dL8b29P}d;+Bl}c16!`%8 zc+|@W-?dgI-`-b&ANMy(rlCv3ClgM|UVI#}CBR8n$Z6+>l?|QF<|%%U$!saDB(o9z zrFe#a5dN*@u@vIZ{3*XxJ`LdZSwa-lKnNvI?79Sw8 zy|owlV6US5wLP)EM#@AgA{gQ(Km|^Olyjvi;vKmz%QI*G7{y7HrN%>ny~8& z8*|SXo}I5gnS*^goBlwr$md;G?efY-Vhj(o$AG@noPj#54|Prnz5~O-Z~{imm%kT( zfYq&Unq&9PEZMkQoL|hwy0dGt(}e@y)h5p?=D&4mf~B=_wd>nzSB;}X4fxc9Lp^nu z+O<8PT_b^M_43Wy#lN+Sf8mm@j$kNeCq?{*=L6Z`tGwS1J#%>Dg~ZOf7$cL5SiD*i z|Lu2)E8zcf#+JB$*T<_#9;8Q}mRy0i^o4KZscN^{0Nm2}TC%H3X(b(9OpD2abh7F= zf3W>y_}Ir9HO z)8g0@YeO;3&IgZd$oO30bA#X|e7xSe<9YA`N9hGm-=9r@zPy`%h`eRE(gx{`UvlPv z`oQCNrtl-Zay%`j#qd*G<2Y)p6b?Pum;07u@2PI;(HyH5Tdswl&QZUBaf;t{BbaA- z-FAU&Kug12k z{ylZ;o2Nbo$!8s>YfUX8Jm0X8G1UWfaTMyj9b50?B)xKWvbK%#k80{P~Gu$@m2o` z+RN9c_AUA^(Ei;2678oQo%Y|GlYK`8?LRjJ?Qb~>?Mv%M(0)ca?QPx2^+lZS2Q|ji zKR!+CMtevTeT#IDjUC0bqx8k$9Tnpm_MUuA;;qOAD9M51;EtH1IUfsgW;yJIh<50C zKW>ev+B6P%&zX*PJ`j!lKf|ZsZQ_qJJ`;nMJAfa}{BFp}|3lu}fLU3U`~T0gUuO2e zAOj47q5=-eAQqISpr$hn1*JzxNK{nR@gyfbsp%V+h`@Yw` z-XvFnerN8(N5@|j8x8iH&pOf&O6M(F}nW}GrkSx zPkwhg{HDK}zKc94j&ANqs^MPYKhm!!*_hZQ)?g)q@w~pT(w;-kcDYLB9p6|!;qEHc zgWnm=#tV#2;r!K~E5BnSq(h7kDSw?l7{&$U6Ts^p=T-)5y!d9C59ab}{zV zSr=ad8=lr$C<$$Iyb)#N5#lS^SFV1N2xs@~k~8@_`IpD7Q0*HV0Ogv7H&4h`t`{ ze9_g6yYl?qJbtCXPc+T&t*`G5xiT5@{VVwLL0#cpvW{#*Z}oNIYy4~c6s@yM44K}d zK49VtVQdB;M1DIy4(I7QXQx^J!N^!^*6%G?@9=kDuiHPgr)a0-8@Sq`=T2z$Jh6Sr zwPd=O46ow58G}>K*e2>xdG+_4GI=zWkK`fiYaX-r=i;*mWpZg6!dR@G7GTV2$KkDb zB^k-*yJ4%8)ugYU8uVjtLQVQA(xFbK9{PpOn`Vq?$8a9+8i3i*Nq4j` zP6@{&*qf<~?V6|9`XJ7hD-Vv1$5&s44c}RXKEgKLiQmKhJ8xWWX>RIo56;7Z$9u<~ zaZ2(x?A2tvVD63izVU@QUjv#;Hcsy$8>fh0d&iDDH5r*|__eow&Z)_fmfo_5iQ&ix zc3pm?58ubmnw~cIG=%d3yuU51y%qQ<=4^*h_aPH0f3X=~ntI*{XlwK$ zJPpURhGw&-CO_%aJ3DM!{QgFb8B48#emkpz@sak45Sv5h>^^O!TaVuKA<8M{82UB_ z9&~a$x>#`$ozd70-o`(+&v3;AwZGp8|cwfryZyMdRXzwG9}+WsND3|xb#n>I!tXMZQy z-yt6DUD`GJXI}dBhq{+P2|dKa;NGIF-&IHtJ#YH!h8J}0B|3i!n0S0^^S5Wm`Z zpGJ-|eaEaNw|#@U4s@@EhmP=?rsG38_e@$FXY??2=FGu3JTQIk$D9KSjfGF>N5~%& zoSK`EFUDCW>_K3^nP|vZL2I~?S?)n0h6a67@Y|gA0WWP_I6NynjQ9bO!)kO6*@Eee63lSfoyY6^=o#$tjZM1JLA8s zzxB4CoJ%=;ThsPxVEM4xwR^=1mG^0@t(CqFAC7y1yx5KQC%MP-i@4^p3h@`3n0a@Vs|w1Ms+{^~6>FO;kJd&x3x~C1-=*y3 zlnrI041Z_>d*I8CK>qUN!|<#Dp3#3cJr2(X!h_rWWHm8R?eo6{ItTNM%+=A~0-jxJ zc&7Tp`+~JsO1__B46cml^ndl0O#h$7TDJ2T>&NW5QO=kOebOF9PgHqH_VL;Ao8$Mp zHpj{PzXkJVZH~q&tLjJJygE9XHFn3)*N*A>gLD~o*T7DHpWxGf&@r+HCJkLeJ?Jr3 zU@x+PhcWJ^-Skb_uBQ6=H&>T=x8&%ws1v(ZyXa2cV^6!A)b4la2j9it4BNIbLt|g0 z=cJRx^LF&gPWar0{t}O{EvEfl@JZwB)!a9~rs1+p?_*uO8^d+y!aB7MJQ`WI3cH|l z0Cv`+3#mKwO=Pdbvk3y-oE>xj?$NyXBjpdpm|Z@PXfB^iWAwlWFk=SeyKAgx^d9sT zE%0~IuUWVX`6#NVFW*G;%gN)|O8FX@F<-Iwt=YKIyN>BV_jdSLFvjAvBR*;Tt-dK< zsSh2pG8WG;hLNJ*;bUd#;K1?!1ilS+E)<%}H6;^iw7LIQAHSGB?v#-{o<%uh_bBI!tqtO)s|O`c3~SK8^36-}H{!0l(h-Cf!xAInQtUu`SnM zzv(CBFXlJ>t^f>u`Az=`?GDs$dKNs2`Ar{2j*IzCEh^WG-^6_80sBp#6wlDjuHQ6~ zb`HXC%E-9eS2mt<;C|Crz(a91S7-ECvwjGDp$Fmn|5tpWKjRz!Gkl@@fAG)ug;oK3 zzAuEmIbdI?M{L>JdD){JKj;57`|i$NSzp%7Jz?*|bx+a{OMiu)fL3N)M&I^dRDTZV zIvhW=F1w}&Um(+e-FQkEOF0}qx^WbZZ{!1;SW2BY^~p4P1Yg0%8!~mdy5I-+1A&j8 zT?_24lgWocet_0@n%D_-YyQ>v-Tb~LJ>F`40AeE_8UBu$Ck%Z6VkO4r?|RbfJ6@u= zjN&E4O?rd>4&c|ACRl?ugMH@qZs_9ik_;XCWA~4RZ)V)!*!?)XI}p2HLjL~R{eNt* zww`#&K+3sx|L?%k-+0NdA0w|5FIgep^waKp7Qf!?{x_&E-|jyt_zL+$MdBr!McX3r zl2>e9E)0Fy{T0yeK<)m|z@wPmKb`(n%r#wu3N@U|C= zSN!1Nf4+_0q`0ACZqU!fCKxY|&W}&@Vplbu)R<6X|FPV`s#uq?Gt?K@ZD*~86R!`? zHC|2pC|obGig?><^e+1~Gd@D+8qPP9JtMYtF*0gkm7F%9>(S3OKQwdJX5LLWm>9ld zO~}R()I;oVhwN9+eId>qgM(ifp8y9FqX_IKd!u)_HnT8Jnzt8TCLRQAT1#*faa7sN z#7O>iCih5e3-8M+MDMaaWyH1^{}Je7nIvjF+E8UhUh`8rZU!cSCc>(KnY-=R>r04Keiz{=mCM-{y|r-OR6vxtl$W zHs1dG|IL;B%ODNb(2Zzo&cu7A`!4{@U_iz^=9W*_fuoH~OOD6Ox&1 zoGd2>vGhP<5QXM$cA$$w|HH^FeAp!)1^;9W13q+JDM0(V#sGkjl4+sZe3tUp!rUyP+Pdzh;<52reg;|>`cKQnpJ zxn6C}j}v3Y9JA1#+oZbQvUT}<^Tv!l<~vTC?=|oQZTig1#r)t7H@7|fn@Y(*A`cF| zYbtcMldTX_uT6y>ha>N5BEKD&KP~d_={TK^)2K>dpSC)YSNe2 z)cp90=`}xYY-O+Ic;+2AUvurV{`PB>x(%Pt;DN6tf7PU!d(}BK?I%&EXo7F{l=8~=;9b4;3$ph{b(gDd_|d_h z`CaPA=neMPoi7@xF8vF?U-SGFzEV5%s>ZKWy*jJrw>F=iAEU1Qx8|P7taHRGCg<*v z%#1~V=OW6m?#P_K=I!z(HMWj^qFikPlQ*(vf5Q93w>`h=+ra+gbIHs5ANyX_x4n^L zk5OL1$Ne5p6w`H~?1t;c(TzEE4yHf&HDiuZy{hw*Lx|zfHpgIn`&INxv3iAr^ryyx zq7AaJowFX2zad?$j9`%;u6eNLy3v~!WX`5z-CD4AEB#mWsp%iJWA>!(;#?8(2}hm( z=b=xF)g8j4*p%XvzXJ{+2g*}47>&h6L*ox?eP(LrUKnX}ja$H32@pEFLW zM9#_6LchA^tM;6BLr*iN$<2Gl50EF@|BB^VMO!;LCq!$Z_U)5=XJePe$|&!uv%T#* z>%HW9=peZDFS+8Jn41%<$!TIOgVrBZ0I$YxKc{a&kIeWW;h`h1@RA2Im;4%SsSS;b z|Lmm8h;J!AA$c8xy!!m_;d|!Y9lYcJ4fowD-tqsZLGNVCKE*r!Q|`NV-tphzzVo@0 zj{i!x-66c=f3W**EbsU~!hLrf@Ay~Q99gxr$;eFWAzVj5pV`dXX3;LEO{a};j4oZ- zSmkZkI*1AcIDyXCF*;OwPqv13KEJ`*CF&SP+mbKERs11d@+9<_;MfzMbwC=s-JwJ9MT}+^Lx^6k zz5mqUz}_dY_wDFwt?L@PV}GWc_^boGV%bM=3yuT3BNP|c4secplX-K-rucilff z7Au`0p32@xu0^Yh)OUH;BPI(UTsc&p9{mwMNC)-MD-KOUy&@Z_HGMmwd%msDx9zg2 zS|j=rYuMGU_)+_dz;?PeHS`hk%d2e7!^Cal!CH60*P9;^p>FaRKjM4gdBXSt8XMyq z%~3m|o5s1-jCXr$*YyXQfIsvFjQuZw!#1svrS3vFxc06rQ(s%Z2OLip^VJ^$%nlU)!wtCoa4K-%WL! zaUeV)PP|QVV$E9zF|h33g&ynt!PA|$cHwa0fBrp;N1#WKdFw*_LFaygu{(U{?0hdd z#f-xo8&_nm+ikl)@#U{?wf12mI^M>Y-{#$c#Fx8NM?d4s?+bAvzT8bYgCA{cZ@_D6 zx1aIlM*J=}zWfjLKx#a`S3}=>1KzUu^%h_LEA?&8TjTZvTdu$H*3wb&pax`t8mldjk$;uKOhMjK1Zrar-KCKG68GE8}ke`p1dA z#g`A=T=#zLmF!-2e#@EDj2?W<(&z?sl5B%?vuu2RjPiBke>pKq$EP@$x$eI`_|NyR z9tCFOgMyEI$wZ;KZpY5a#wn&$$d?TLrtqwQpD;$&KG_?XbF8uZNf%;^ytc!WcbI*$ zBa+i%3(s#Ltv%G|A|vuWOgsfTn{$AetCk%7i;b`6m%lPBKj-k|nPGW+FS7;;d0w*@ z{0wg19?qdPeo=lMcZPMW9g%#GtwUoajin~s;U(8mpYGpsV=L}C4#r<#9(X4$WEY zzfEUStDM%k%FY|VPvhZ}@Oe)Hrtk~`rMZ*h1M(B0|AeEbr>k!v-$XCnRON3UP_pp+ zbq%97H4{Vq8R_PRqqyIL`?~o%bLlLOma^d=OP{r zY!*MojyqNcIGcJbK1P1nUnk$8H-F{DiO>gLxo6~?aRG7g#o=CTrwE7nJ<{% zb)?aOmWR+s^hTZoT)rw>Ki`}$sJ%_XC8zJ?`Bb8B3H7nOa!0RzZ``3db>ze~J`N0a zT&I4E&e*N9Tr%VFhFlxd3@;gzn=@;5j`C`7O0B`?qz>I*E!;iUp`3`E$nR7i|99a` z-R;s{UR83M)=w@RovbAfeWOymm!4M~sg1h+b!?!|#Miecc+19kV_w*I-A~vv8tet> zjt@4Ba(Hb>|GO1`H0uGTZ@*T4+V?uX{Co-P!B znA|2_5$pu<-qgP{!$ z_whAe?QI{6{0l$qoQYX=j?ePitgS|twbrc3&X=LbmwNY=C4V7W@Lp@VUQrq5M#Ljv zmQCQ_?D0QY?@XBm1Cq?;nKJHrr$@enKV)_A82Svyeq{ZS)@*FyRobAQ(3ULqW)4h# z&+=#rI7>z*hxb2&PrLf_2RUc!-Y(z}vT>0FgHFhbX80ofAGw%EI; zBo9a1dZV1r=Fv!HiP=?p{=yx-1)P3 z;IAdPx41uNZV1NwGdH$4XKpM8CbjAQn|QnK8V8==g)y)=uyEHYc&$qYsU3^Ta&Ra&$4(+`6#^;~kDjHfk2oA-8MWdX)c71vBP2zWIPjL(% zIOH!xDE~C&_1@H70N#fjcnzH?Uk%Kf*X6gxg?(EA*u}?l)$h^|9o!tSfF~Kl~W|K3J3f0C2Wl6UcVz8hE+_p03EA6TAoc z+s$)o4YD20o)>6z$@9Nj&fGgL3&_v4%)c@+FOieH`gr@KsT z{sv^EDwZ_5B07w`L-{|SJVTjFIKON61vU;0+->v0g}m+aSfeS}=W*X*HF7pz`f&s_ zpW8K;{8~q%yMBG4M%IU*+;0W9Iu7^V0>%e^mxRbZN&P&%D{?(-SjtBP{;LhEhjnTPXBZWKhIo%U!a6j9^ zb#@l+n}z!W^uIED&k^a530KzeNKfd_qRs~4P)R+*z+n!*vwpjn9(~{Z!4Lrs|>Pb=2Pg4LD=N;4m<#UuAEEUh2a) z=y#yTA!WHfC~*|EDIb6F8gx73lhDtX?vpQZuV9OM_`ITN*AJC{2j$v$&RK`z_%fkR z)A>@@s62HU+^Em^3Ba<&*0ub&z~*#^aP!Tz`BxT!>#`iU0>5c}Kj6}wGGoOSXROz9 zuIggVJT|H&5%=DAg|&Wc@K@`zp>6UweI}}_=ZmZfXZpUj`MI>pXya| zvp~RW^#f#EKF>}&#*;4SYN|B<^)is^t*{!`XRmrRF!-;E0;^3DVn&4TqHrZ3)r?Pv*UCLQ)mt1kpg zPG2gZyG|u#hQ2w~xz~-C||+3%tAB_LXFzdJ5=-upb{LSn_njLeeH~oI2>QOu?3ma<(KxOd|&*Xp?cA$;wNw!U*U7q+be(T)W2h#;pelq zH@NFf@o(FIGICT6t=q6+SE&tby833=#uc=oxVmEZyOC+rC)Ib+7a9xqrK9uX5KWF< z`vmQo{s8~O_C6$>c((o4%B-{Q;vi)93BjF5yNP|#u1GtfoJeL<0X>X91b!p4!>Kn% zW{dDZ^iC9@cif@3V$45M8B~KYXAIoU-(9qniRB^swYb z=3J28$AF@B2>*uzdK%ftsjFsi^24eN`N-*C1#tNT^D+6jOzI~tzb;UAOrf&9@ZdSs zk>`I6RvrC-eM5ovy#s82b%FL@?Bnuc$qQoE0o`j-^YC3h6&$cETu8l52mL- ziXZ6tU%o%bzc0X((^PMseEqzTY!oP4U7+ma{Ve;%0%h0tv+TbYDEqB`mi=UbvR~_G z*;NH(;P!r&{Y8PY*B2@qb=C`E%XL4M&SrR$SiqCS0=kJuX)e57b83sGr*BXkfVoVg zM^D8!3hxaF^ISenHhdX(DF%5?!vD5+nrq*|X#7u&>-DSqIxm#(8kXgp_={9_k}W%_ zre$Wt-Z?{_i#2v}#^j27%J)dHentBc6`Kfk1#yBMdM7wfaodGji)qRYRrothQ8ILgZpXRF;#ZG*8Sh^8Mh19_P*e2egc0avscAg?^&Qf zZXv&4)41bD{MNWJf$9;82`nA^^6=zN?vEfQu(ZD7@Z{jEO~Zc;Vgl_$^I`&ZZd}Ul zLqjH?v@&VzgX%_BH0RpbdQ`ACul15miD)1zW+Dorb)}qJK8iIcM=|CpO}5qJbIvD@%iK=KT<$5KO&^&H zUg0`1(kXZ*98$A`w80&orOqR7T_m|^EaQ>$C{q&j)i%-zI_5pNv#yGN@=ujsoE*eBdG?Qk zCasL^mfQI(;MM+U_3PK!_b%V<*L+h9v~31;Xx{NIujOvbqf|Yz+2AMh=aP!f%ZOg# zy(pwR>nWQ*C&#>z=3SbZJ8EL?rP>eZG*~nye-raq_zp`G=I#JzK1=g!%(pc&$JWH0 zTUgG}K)eatCEZ!4cJtCPL0&k#xqvXhh>O7S?Onubz zXUdy2b?ph$GiuVe(btQ`r_^U}|7-X1{C;~Rxv3mYUu$+9PyRd(G8T@Twt18J*%Y45 z-HlBhdcJf@lfCn#spHZqsaP;)As_1Nf@z?)pE&e>!9_e^KXIr1?4{X{E#J>txcz>` z7mo49TyczVb?lk|kBrSq+4&LWZLg}?uYSXvQ6K}@qPB*o&+z<(AJlvjaAnR&(R>0m zn|x|p|B1XjdY3)FBgo!e@{NtDj#^N%@p{}4EaFbV(p9OV-JYt+5HZGT+!2?)%dzyG6}st2%U5E!vf=* zZX3ISyUik2Kcq$^7mNvbuDGEscttOSEMXBFh~rKjv~ zapax7hRnBHS%**5!+*_wuja~Q7z_Pa{iDgQrDmPMkG;*u9@DPx$UnZ&9K&Cp!*}2t zyd$rr`H%B?UJK3h<@$_5azNYFC0RN6)6wu3c{XkKA_q?@pCbp0qk&$N%ow~z=ixQl z;-&8{UOcle#k|YlMc!g~Y0aDbxUbT_U=08Ban<{5AO3K2cdKt~sQ)e@u%)QmAg&xw>n>`ovL*u>*QzDjocKh54tcI8(d$PZqhjQYAXv@*R)g* zwBr)|k#IfDwhHJ|Z}ZeMPK<@)+__yH`Zme-p-J)wwcd%fPB$JtMZY?`q;sx^jCDO` z@ZeqZjTfL_%l1g$vfe0=!@10j*}32x{BJyB%CCh#WtsyYi@A755AuCG-*vakgW}_O z3(I)y-FetMeAEPEh;}Py)0L;n=BYAsOdAE)iIxZXZU2aGsv|W&GF88w^QTP}?4c~@ z=l{O)lcyN0S})a9cj2Zj!>pVF_fuN$s{Supo@MEA7IYqJ?;In~a>32o1i@)TGN}Ors&u38C{>@Gw+(;oR{UZV+RWI8T)|^K+fk)G50icpiT~- z`Tl;sYdrTJ(IRL3I0{(inXzKX=by8l*c}&ETR-CwyUsX=Z`6%%#yxfOz~@NtnFs6_ z8`ul-nf&t6o5)iPR{1XZZA5tYulMg?F8?Am8hJ823)VL|>!<>K{NMveF0k`6W^JtT z^U>8k?h?MJ>nSU{hBu=npVmj9Z^&N#hs9anb711TV5$*JH(oRad&_x2bMU`sg0??U+weH#b?4~bVf~Ks!@pv><6C@}?m#aDzK)|i-YYseF`mUkvbtlj;0|Lu zBY-Cyn^#!fft~10cZ?PfU0*rmp`$xysQoI-!zzb|=#Go^ZLp==;GVjp`OOu*_O<@% zg>k~O554dVYmE!(h4pp~a!&s!s2A?<71o2*3-kZe>IH1K;wIs`$9JO_9=CF7MywYfc*~2*#3A+brHYg%aW-AI?B}@$19JcUqX9ySRuKC{u#exl7EZw zJ0{iM(x=}+o?@`}=68%1EsOXaw+QZc;dfMs&wcnEQ`CMDzvFCuE97@{jJmm~-_bEI z%jf?39h9-QJeSXo-!WLU=tIA(H~ZJDt}D&jh-JL*(a&7{(s3mC;CFOfY+&!z?;uYx zSc~}`GwwU|e#aUsyGFkh@H^&OoO|GH^~DWhKwt>5u!%^&vRcicY+eirdNzNc>m z`gAJOAMfb3KN`QIm;M;~9iNrFIC`Ow-!VdY3iut}M--4L*Y6mN?!f=(9Nass#_t$h zTzB;5cT|W@Mf{FW3hsB|clhFAAAZLKwO_>VI78nG`5jhwGki*qr(aFk%m=!HY( zcWjg2acx)Me#f$McvZyjcu?O8`5kqKZz}3{)LpRYfc*~2SXu{Ep?U-&4F}x#EL;#5>4S4Avrk$2P?~e9^Lq-!V^cADrJ|`1}{< zNc!?Sj#T?a{EqkNTOq%rb=0P!en;!PET8-9cTmRK@?1VUe#aZ|sxSTWHRUPo>34jl z5T9NB(t0HL;CHlMY+&!z?;uYxSc~}`6TW%q{f;j>`a|R8{Bg}xi*qsk5)(`r{c_0s z4kLGSBzJxI9k&gHS4I4ed-Sb9pJv~s?T-t4?T^Oq=%qh~e#b?Ump=TC^a%14@H^Uh zUx>Duc*kA9D!=2d-eEO<$6dvBM-jin=#IY(0Jb82$2r37;QS85!w%*X`|>*micdxS zj$`z#kl$f-M|10nUi+HyJNls)*hkiXB_l1l4-~9nCZ;3d|BTL_#LU) z9y{YbiS`4e?alkS(2jyddc-Q{=}X+SEq!&#{crZ z5Y5-9ZG4Q>+012}T^xr0B%k7*)wyv`qX+hu(Wc`=B%HYCM+I{MA7b&~KIcq~9%yG> zi0eaSc;>`C%fuJQhsf}ZxaU#&R>+57&5Y|qSKp^{^Jg(Ul;NRK45VvB!lYrtm^!y z`2({gLykYNxU^@#zgaTu_ygHBzH__o*0%!vuCG3`PBK8hoiG0OVduMtlc#{4f0g%z zXm&4kn|X~o`fXkDe*0y~iu!Fu&U}UGw-Mpx*zd&P?0m%~g1LbGUOcGJ`3lo--+U|A zerIUz^xIKt-?85rn$vIJt8azuxB6{A`dviu_tEcGi{=IT-B$|H{1EiJ8TTG~d%MQz zSB32DT#HK~8B)JHRxoA!@H~6#tUD8(Qp8dh*!5FPJlAU0IXeqj$OXk;jH4 zhyN+qljrXB3(xNR`);=!`<_mZ?{8yKA2Q_VwKLfBA5YA^(-V!Ic`CeEqcm zuKw&7*Sv53esTQw4+t);m#cH=pAYMEz?#y27uNVD+GkAF-0o)1X3M<$L-w6H!^@t7 zX!gpe4Ed|uOQ&YfRAhZ%p*(+Z^SFCAv+Ln?7sM|5(;w&)T3e=b$9A%Ypo6)<=b_i- zcJK9@&_?UG!tYwQ81-w?&jvKjtX+gZtVJun_fKtXmELvwVB+67r%G#4*zd5wuJcm9 z6{>rnoww9^7`uwL$gn)I2!g1>8G$<*UK zKjx)7AIHCWjIq!ppEFo|emO60Vr^PD=p3Ck=G1lOPv=8CKd685uVD?(nws>Vrh`L6 zEO~|I%(hl0XD_kqJ}HxWoxFTY?V*j=sUJTuwMYA1OOv~2&`urgo#k^MRc(56DLB^i zTwe=csedl8omHDYm-<;Zw@zpMq~`N|el2%r)uizWH%wbummd5&?a#9@1=jR-ww5J3 z=aFwN_0Oy2x0bcQJYUW8H9TK!@jAVhvy4guyk_(LJpJ?iY(L$8c5V7G+BGta-`P&S z_Vc{-^*^I6e8K(0?Kg0n3%;Uh^3VL9ZQ;OAWIednTL6IDsY?xtfvCYNdc^aqn$c*-b@>1E@GX$-sw3t5c&C zM=RK)p!oa%XjE24n`1ePmi-=DbEAC`+H)M9&#{qnZdH$|i?MJbP8)R=<~m@h=RDRC zz%vYZMgY$+XgQkaQ9O_4d6cE)VA5yVa?A^-ZUvuP`RxF{e|X3Wbbg!PQp%RrnsWxP z5xq{gWlpE92JwRL-IR?|UpMc@^ISz+<9V(M+Uk5WsCPH_iLGOed9&7-v+krhswp?r>!UpYS_{4k`?#Sli@U*J^4bB6g83C-=%D>cJimgxww5G! z-wGU`1)dwB_h)O|tMI&n=LdLRVQG9F@(LWLAKu0H2L1E>cI5T;+VsFsUMC^1~KafWy$|ao~7N z|Ip`A=<{f8y4CXAoH0+n_EBE?HqmDk{UoFhxYdL2F|u9vfSWl_&Tr8>c}3WpdwF z99}#p_~6B}@Zve>G??dS;YGBBv!~hr!FgcXo6*R5VE-mwum?ozXACcfX%FJ;uD7RI zSskYJE_`P#lgo=u_$RK9+gu;r-mJBg>`iIb`pGEkBw4fB6lL8ee!R|c6EC#?qYXI3 zi)PN1)83RObG}+7eQ!?JZ|?>8Ub{!*=gsA3dWjci+0| zmXD^j%I(lu_P;5l9qpL*!9l*j&MLdt0bD)}-QDw7wBI5Bd@rr(y^DN;*Igr>W4jwh zasLec3f+0$3-GL?G%3B931ON{_$Y+Gss@6Mv`(`o-K>OVbb-}KXtlBD$U z74m!bRwl3Ku3+=Nj{R|b`+Ov@(Mq3M0nd0Zf2i$sXwguTEQ#9hdaf%;j0_bGy$#Y+}bdrpn z4Xyc>ngPAfhSv3zn}LjRR`c!#Xns4izZDs~eOh{oWb6T+AFwiZHMG2STKcE8R)<|p zyDRv}olR_D*mkvY5q>p||Tk~tu{{b%ypTO;x;A`xH!xz!xXT1L^^h;H-9=@t1 z$yt5p@3%6ece4L&*#579ccXk7e3{g|{`qxhVnfxw{!{a;UpLqC2S2*AqdeJ(on}uR zI3wrS&dw3wHw+q#0LNjL2J$~<+j6s^@1xk7NBLa?e7CARzRPF%y-WXrU-#EN(BM2< z<~-VJAdh?%{FF}Puh7?BWUT4yFu7FM*z*kTC4*Jo2ey`KN_5Ve$Q*B%FgImdS4d1V(t!rw{ zIIVLMZB3-DNwhW5%E>&wQLm8`d<&i5svUFW z>X&@~Wkx2JPD3UZ2*;BwjwgX*-JoRWbL4v#9G|P@7kUl`$ARED7#s&$dOn0q0FTi< zt>k}F|KxuhnRvW5T@lKJbm&=p6YQr`=ULQO<-rSd=$f+R?#HP2QR;sf9s20B^sCa- z&-47e)uH2s$F%f+gm~a@r+&%zUuJaZQjM`s5grpQ9utMfaPT1Cv(*3GG=8V0JAcdb zi#-39=NCgf(4oK);KBDN_0RXm(V>q|OaI>TpsflWO1}2dUV8a|)7I#k^thn`e?*6| zq}$;6ZHxIu=D)AKVen~`-bdrf&*=S{c=9IwR>hOw<$L!%rBhA+ZzxG}p|Tk-pCEtQ z(ZkxO=RQ~EJ?jh#_xtd?XPwLEe(#~@JL1W|3<>dwCC>pq0~0uW9vDr!6+VY~qVZ&1 zm}gBqSrewK;>jDr{B=O0seivcfmk z2c|nuLRJ_Dq$VLNb;!ydc=sy&=|EOqotC}?S!vWc$BeIxe>(zM>0nRlSytvpXv`3d z@9dbt$jU>z;lU^?D~zo=I}T?bJmWmZaLtVMni#`1>)dF@aLtSpni#{G@!KoN3UC-% zc^f$1)IYNFI*w%`Pc9lOKAEZ_ zhIJSCj)w;0Z9gszzAOJ?>d-#N?uJphV+KcFMAzS1daBH=VY_d0=Q#T!yBkW_hcc3V zC`UB!hfg{~^=Isx%(hwA?SrSHg zc{s&j%oq`Wrmc!|0MJ2yJe#_%hBx?dWz^BcI7#s;*_6O8F{Xk>ChgZgqI4Vfsr_`u zTiC4*Y_k0L81dj^B!8Cgx<5kuw}~q;uAuyUXn?<_GsyGL{Wda)9y0PbkGymBPu|%c zV~U#eqtHm_Z4ldD8s3YjcpkC*IilA#_`m+C_nGy$6JZq=PrdOo^q zUGy@3FYQ{#n6;TP>pEh{I=4i3eCXWWPUxet_CEZG{kLZKYP#doPU36aM~)Bdr8PEj z$ENPsSg>jQF8P^5ecFF-?;J`CezkKh?+v`bA~*!ES+B?5%G~j>_QaRbcOvFIZ=L0> zIf$?iJa?vI@5C{|PqH>M9?Z3g$BJ_X`w1`YdP@1Ajm}3>zCd!hyS-cSL#|ig3;eHIb2l69&7Q`K#SnH9L3Pu|oB67T46UPSI2QIFOb1rA9n(B)px9o}Joc_xsY%w|SMr zlhw2l;FX22;P24$+;#_km~77d#_VGwobcn1{Z3H~cpDTp7Hl9G=|$2k-`u-2R#i zqhvrbEIGT4HjG|F7F^x&i@S5@C&TkP46UGp1Hv*j!hw{M#9mgU`~$4Rz)`uv)v<8~QHwypbBCE*(2&Wb)3ch3D6kuAe?KIRblN zXh`{|!tx)%cjr8@V&zwd`XCVrF{lxJ=Jm66SjYsZ1ULw zZFCQq#8an>x;>>>ld)D8HsGe!+M1%T`f# zyxI@;F&8b1KXENGqxN=-&&VS7JIVRHv`bmy48_Wh`(XF-jUO@khkhT#!zTz&Y{Zk; z7}+PIJApYNA4c@Tc5uc-a7Wvzyth8gpI)^(AHA(JjY6L-Jj1D3<9uR5&CFvdE@bQ! zy4dJw#Yi*;CO#kfE{lLoege8IbEmWGyR4$W_1IJG*btpZruh<`k7sN;Ff;#a?lcm= z?*wlXmxo>_fb#@oEqlh9_FV^lh_N{h%$?Jhbk^Cd&cGisGERLD9v14>tWRR>m*SbD z@uA4`ZSt7>&}I@k#?`&@`xqCrgm-Gr0~)!zaEd-&1q(HR0Kf;P}79R;0^bMo*x-%(*?i&DWdt4fG}Hii{1& zYd6%9KV&SZJ}rD@b9>W;8J$eN^F&|t@g2~mhrP|yp+;xKYtsIh#vR`Tmk>Af)dp{} z(N`PB-aImStzncQP_zRGIsg5G|kZM^Jo@pBK|c5cgWXX6pli>95`71?&S zhqi@wc68^#=9GClER#4g`Ai;cW$~J?!#8tJY%1!}&*?wZH{l4@QhdH_MGnjkA66mD zqPz5-^n>iXbezuSif3$du5FUPWqf4H7}+F0vS56A=@Z$u(Xjvq^PE7RhO$?HFM`*D zH;d2{rZ42#*uQ>RXRZW(MUFf*Wc_P{^Dx@1$G;OS=qZgWWxKy>^bhm^?oRov_8Ut$l7p^-9&(;k6OxaWUfbvnK7qp({ zxzhu=mP}u@S?6UrezL=ZaBL#`EE{ZmX6g|yMnlWi?b4s%LOti3dK`W@^%U|wW@|1E z{`A(T^j&RTS76@G9UH0sOrL#~wN8{XHukBp$qBVyvdjnGIR9l{`X^eC4?g0j@+oFf z#!*poFxLx)xPoX%^f88 zPG(J+aQQyCsLt?T?Uc}t+8s%|VS64rDB|dfhoDdXUA8BvujKWykiYRl{4J7qzyaiK zE=2Dl^>PP~;mu-T&(TQ@;!m8hvd+-r4g&h9)>DhGk`47KjnB@69);vieR%@>kc>(G z#GC4<<=?$t@^RW#Im0vL*pBHB_#R>LqK*02(ymW6y5zP|$(wdsT?t&x>;*79g7(4PlUcic`Qp6wgwa9U+a3kx zN2$Axy5S-CZZzfV0^CxMdFf|0M|@*EIUYFqzSn2nX4`e~+wOQoRSpPGaGmuh8yqu|u!W_HPx-%RRHS+P8X?yjs)M)=`r_<22|rkl63QVE$<- z^JS?jJ14(~x?ZQQJ*q3X*DyTCRCT3Z_vYni!$SaOX`YdvR^rM2SO?Q3Gm2g7`PJ5T#T7@L^38LOx+ z-Mtm!cZKoCj!J$fEJGcdkI*`f_Pc^|s`nSXQ=e^bWzFKP9`nDz6pbYjkY(Hr*<$LS z&lx-;f_3-$wnN`0+O{WJT3K996kOx&xADOBSudRfQ{w34FDY;8{VZjtA}jT*uY8fV zpNGFMQs#N_H}}5BP*ysx<9kEY&OfuZj5!O6OT@=u=t3VuWP;k zSpnVJ&#Gw&^FC(l2%oPIe`Xz>yw9b@2cbn3wD={ocn(_p5?r4XEi_Kl{OJY!e%y>B z1#{=se6I^=v77ml&MN&gUo@KE(OQ2O%)RS8;9$*V5Z`aa?=xq{Q&J{eOg&^%2CmORbJr6CoBGPu6NJ&A2m@v!SKh&^o|2zMUD$3+W?E z(d+f#bsqdakKcxfk?q~1;Nb}U!_Q&-4ii6#so2^h{2gyG#yM0Ct^Nf)sF;T+< zr9UMc23i~jW^s5r#DP4&TNuC?;>UWO9iym2Hs8R7UC{Wktjr7io5wseC&fCt7R_B> zZ*i0@ye^!}eT=a_>s#g5h4<2b=}U@_%C`&7M@><`I|t;&i6yeu8yXn@KOCcJZb)-G znllN|Orj7;w?pTKv=aTH}^cdT068PTd;64lr|T?%YjJnBK~R>Cs+b`r8+Kj%PwY z9=wvYpB_n@qfNUGE%JEc(k|jJ-0{YPd9>>#|7!ZiANt52`lK48zs3AjUwjW`HpOjiTZQs z*^sL%Liw60Kg;AT-Z#R$;n+M!rxd9-%==C1&&i*MgT|j*#3Su*yMyrwwDh72cYKX< z;*aNV{-EbA{9rhCl&(qvujp1?X5~=nxOgzX%e;q%F77#N%XRPkUFf?8Vw3XGTcgP@ zJ`TO^BBm)nV`*JFB0G1T_1UlEt_-{9KSdtxVd}BIr-U&7C8?12&=Sw6*M$TFz$?kbx`f0U4pZ4dGKT7+PZ2MgwOtz~1NkRK>i4LUipna83 z0h87|HR|5?Q?$k)V>z`Qrq<*ILFREMd zRPD#nyz2?TEB@7JO!?Y?-^2AtW*+NyVr}UCdt|Guf3I}{opr$a z0I;rrKMw%Q3c(uqoB4NxXkM|cp8VLzLigiL;BT$?dl&Fl(f)Gay%%_wQ}?}sm-#k);QTvDv?s4^MooIQVx8G_ z0!zJ_qmwtdJXgI@>b-}0@21{+sP}Hwo4ZabgA>2NFEac;dLFEKro8*;$x}rhQ;vMC z-kP#rI_g3+@Z&eM0-fvDx5cXd*iSdnio$tbG;@{9YMP2e^%{TZh4ci1`oqArNB3kT6HQ@4FD z{%ZamKl=@6(c^oMa!x->H85vNAJf=L>oKqi$P8mwVr^>cJ0@<;7#RE{k7m6nc0l*{ z6&nMqk3VuUy3f!zEA#n032DoEZtxcli9Kzs3F|>$_-=53z8TzGv-(E9x#6!nhF0GO zj!;LaZzuBNS2_em+Jw9f_*$GcAx`7fzO7e!*+-rQlW>$g zolQCiS1bF)XfFRg9HWFb=lmjd1U@}@gQM2#lrawX2L0^*Ox?CElP}&&K1ZJ$zN61G zJfNJ$i=vmtZt81lYqMRSCO&BXO8UdBAEYm%qI{WnNH7&B zqqv!5t6H=X-n6MTO`^-OmM+pE;@?o<)OW=al()p%4SjnL-=qW7u56X^RmX$+ht&a;hplq8cxLm3_a?-BzKhUI8XKAOZ*u7D*5Hxy4CQ^gFkcmrgpSmSL=NZJm}9$ zdGDS>V&VZBL*b_>@5jS(!24ThkAdP z>Oj_RvGBRNs8AiDj4uTb(Otf@)<+EBe;~B{s6TMeK=kj~gPyy80C4Mh!k__nYt5=` z&`Y+Q`0P>U1>eMkj z`;>dseaEwZg_q<`CsUWoC1~G{1=622Z3lgHBYAS@P()52R$ZolIdYn3L(KSwwnvbc zvA}leK8+J@$43b5WqUn*g+8C)o%D|QDf~q*7mklC+)sS;N#eI&lD@RfTXtmfe)$@C z@clPu;wEGR6I<@j^~HM1uEu8_v@JSFm#3m-tce-jb=IIjMu{gHdB(3Y;|%o~t&_)> zu2k9^Lfi>?7k<*+;-&hi(c$D19`Z+plfGAj({A+*WTLuEwu67(t)mX}j&^&_3F`FI#t_)c>esvkP48O!H^U`a<)7W-! zSZ;A9hCRZSk9oq`fzik+u!(+NT=%MGeZ*m;g+tH>3ftB~b0HJO2VlstuX$~13?u$D zR(absj_J`xT_7i_CnG0L9WHHyeqN(_ik|J-dZ3Hxr|OsaHr9-%^j&?H_VnHSqSFht zokKI^U2_#7-G55DAk7-G{>5hIa5Ud8xD%{*ZfAYw49=A^^4i_K!CN>Yx%2PcOQ(5W z^2>R1exc5Njdu3m`c#t{BStlsR?0nUtSf|H>hD{$j*a<p)Z^`Z#d63 zGZN@i@!+f9B9D#Dsvmw?&*)`i2hf$fiH)I~-FWvlZ~d{!U%sTfjxXzKyvy4@o_n^H z|4B2ahAxfQv{XlLHg}8Zz1a(RoctNi&BOPbY5R2eUNLRmOPuh%H_Tm*qhOFvtonW6*ZTC>{Wb4~f2zmj-!F^~aBL^CtUMY1A-{%yz^V7*-(Rt}8UFEI>*fvrNGned{!#V^ zv}17!_=mi?V;dj-ZJ-ZUoI5S;MMnS1q|bNK7dYvQo%Cf+`V&t2lTP|mPWsbM`ZG@Y zb58mTPWnqu`YTTQYfk#>PWqcp`hF+7P33pF8PaIO!cu`d3c+H%|KZPI{-4 z{-cxL?WF(gr2pci|K_Cs?xf#w(vj(o+&bw2PWmt>J=94b?xc@((#JdL6P)yWo%H*i z^eImIG$(zglb+_JKkTGG>ZH$g(ib@Ci=FgkPWnnG{Rt=iDJT7DC;b^G{W&N71t)!{ zlm3d6{+g5iHfw1@oxIvff78ixzms0=q}Mt5k7vvo*7HLr&ySq+Pn`7APWq=#`sYsi z7fyPIlm3;H{*9CVy_4SQq)%{Q`=j%Ix0C*}lYXz0|NTz-6eoR}lRnc)Pjk{AcG4ep z(&swq^PThsPWoae{TBze%bfRDI_bYTc|PI1|D==tl#~9ulmFAs`_DM(&pGLLocxgw zWqDoVqz5?Z!<_U`Cw;h+zSF7yOHTSLPWtOk`kPMrekc8HC%xK9f7eN`bJCAE=^r`i zr=9dqo%GM0^e>$B4k!I9C;b~I{d*_9(@Fo)N$+;je|FM;angTt(tmf-?>Om*(;rKm z^Z+M)n3Eprqz`w}M>^?aob<6y`gkXOf|Gu~lRnK!pXsEhIq45O>GwObaf*{Z%}Jl> zq`&W!{j`(*p_BfMlm48O{(_Uf(@B5HNq@yjf6YmM-ARAbN&m=6|I|tU+)4k!N$+sd zzjD&Qaniqc(mS2>AD#4WC;ew9{TC^S=`4O-2i&1as529Yp zzs0@lhn9GoSPY zV~c$Ig7(V2S)MoRsGGcNXPSD;y&BJ}DZkb`hhJ0vjKn3G^2$HP^Txz#ym9*8?N@jU zseGXyTkCs^d=88-WsXWr$&?WcCnnCxd9F;Hobx;~HaYjX7noI_TYtAd$Xnof3n~}+ z-fic2`U9>(9+Tk3n)&k5eCTrMs{E-&8*L;cF7nsX5z^-U$z73(eKBuIv~o$@Ym74c{z6nQQqjt_ac`eqY>4T_ zGqK7i;@(TKy_BTBk9!w--h~xQW8U?N#+Y|o#fF%--mhqj!P1JoG4BPxa$%eug5|8G zxi3<=IZo5%+u~lENiT|)r{W$*k@0?Ow0vQS_fk|04`3?y#(Cbj@^ie&HKwg*A7OcI z6c|CLm67tT5pR8DBrl$cO!mCDA}8xxH1Tqo_j2@l(%WK*w@Ak-7Y*>%#w&QUIbP8; zz}pwE*fhY~T=GneS4#(0EFI!)9azyg#9LafOzX=lUKrw~$}4sa@irf}B+jc{!zS-K z(rc`ovhPT*t+IT{C~x8L*CISD9T{6owIgFsa3%4P6}(w{>rYuOJdh%>wAxXthWh5Z;Hg8iTGP06}zI|+GuQJ)ZY|~ZHf6$#F#TRco=xP{X|XT z5^pA&0G=P_-R60>O@3J@(7Ji&elM2pe>uV1TYvBCyXbR<=be#Q;7x~424?l6aV+Ip z>Z2hn&2sxug-1Y)|2gY@?`40C&C^$(pP9HhOIJhZo3hWx`A1$0!~NUnecOC>$Gv{6 zDdNB7174QCD>C`$O)vPdHXn@}TO0BB=H;XQF|iz-6rVnBeMZJam+FKoCxg!@f6ez6 z#J2d}wMH{Qi;9`$E7|JgC%_YI4Sn+a+CBaTKk>wHZw#Tr~s`h{3x=`gP?R@peryDwhBn~m{`XNGxu;}tIq^EQ^~&9;(?#g*QY(u(^k zy+sMtzBy68wbFYjF=bb!cVAigTb16XvaKaNG*!fw@Oj8>q@NhN6te={u3R0MEFCJE zZ1xkoAc1JIAyST}+8i0li9^} z3k;s3j(T|v9eq{UX9iC%C!Bk8Vy!w3fJ+`LE=e?Eg8T&bb)%0?6piw9{<&wIytu@> zErG3C=ATG%vwHMCzhW13j!a>f)RxFLD12W``L@O?JPcMjgnunIk{65OQk$fhu zW)EdQNK7fe&6|$K0hh|jiA53blEgDO8i^Oc6b+&GBWG12=1cue{$4PMmUl(*4n!2b z4f23!&dHafM3Dagwh>n)FlR6LPx#og*P`WHVqRAi(kOqqhy9pRe!Vvx_r^TW!e>id z?_DUnBHE3NEy>Ykeb|8boRc?Z^AGV$=Dgr9iiX)HyBAW-9>P=q@r<*&P-+hSGkA62ew*OxjaJ?M(bp#!iLCTzy?~P&Ng(^1A{B3m ze{zBOqkkN;jB#ij?R@!t40>9r3_oH{?WA@J93NS%$_?1s+)D-Cbo@0C3zdUxEca`j8et+vr z&V|dIlQ(B{tzsI>XcM}irdImMDe`(iKJ*mEm)2Hi%LxwGpVo6e?pKX$%)n9TyKAeu z{UNi;mqlq3rVPxrb53q#XSJWWFX6TM7j8&+ zTO*0h32$X|%(jHr6ic)v2%#kQCcIQE(OBkfjmI_+qbP|XyKN;LmSAxoIcwGiIg3l9 z7@A$NO2Q;v#uXP`dgvGNHFlvlU4E48HC`J3FGR8RTYK8;Df@hHYt~*@EY0DSYoA4v z-gRpbhrowBCmd(r#3l6VHbL=rFH568;MvOXq@V9J*pJCV5An-kIm!*JU;22zbhVh?@O`Q7$qiz|W( zu0%l=TfLgaRWOW90s%dbRb?TTEukC<=N>dB#qK$ixt(-%jHN4;f~ zm*jq2zONIq9tC* z`7_(%m8lZ%wRobd#OsPzEH3qyluWs=l&BD+MOJ2R^!H&R%C2Rp@2&ym?5@~1pki^k zw`8C`!>hq&UZBgz7u^+s(Rw$bMgjV3pNSDZSXPoCy{V)e3$&dYDU;yYegzE+{N ziiHtxt?N5pUjBmbHTfg)IW}A0@#vcqTQJv*6C&OdeuA4)7DmP_;Y~#SIOr3-(^fdS zdJkPZ>ty0)IHhg!>tm!h_*dDwE}wI=qc{l?^`c{6TSNEd1cf$T|X?Bu7??Wh5L?Q!syn>${-KJbyJLi z@y)16y(@%U?_;uqn=@BCz16Gl`Mz+E=RSPO-fWv_l{;2HP`&eR#{*-$6Fu+5aysj* z*h`wzh~XEk#V?f7R7jY9NfG`-lstZypGn5=$b?7pzb4uy25R5Hub< zS$1!hCLYk%!MnrGC||@Ju=HK9h=hEHO<`KZyD)Jd6J3nRZQsks#kF<9_5aV$2SUf7 zf4cGUq0Hm58jLv*hHOu`F%@-|mHG20iEKzV0#AIQ-xOmU*c4|nTKtpji{t-?z4w8O^Q!Xy&%MtO zhLpCnr7dl#K5a=$TAGlCwlpP?w4^01NiZQTHOUVDCV^%Glo=9Y(-m!WQK>~2m0EOB zsiI~p?xLbam(}Rvcd?>HmsM0$s<@(M6<2g|`+MK#nPCP3fvCUz?*4xA3i&*L?w@nc zJ@?#m&b`CL=A9#PLbu6Tw_C>WQtmiD5S#i=ezjYrv|Pq|onjhwD=W6su)oRZ{bLY~eE*j4P7Za~?bPULFl4c!yBK^Llju`7(P!hsCqL zi@oghk6r9#hkN4U-nrb?Vsu^+@=Vc+YU9*{46GW}#6q13I8JBbEh#Mtqt&E5WC@$Y z`HXA#dpkLcltk9-q{`^A;3}C+&1vl+Q!}9Pl8g+@_{QcSjc&)rw?_8Niom(9S7xkj znIDVBt~1enxPFtHJH3n^dFJKF%z7KMFK2j&y^rYk*>Z7)&vUU)rjO#ZuGk#hS!1?^ za&+z&IR|&;n&-*;!iob;||NyZ4%Kb zLT1A=W+n7nX%3l>smJtV?yApuIpK(lr3~$S#+Y@R-^7u7*~9B+k?-*oPv6VIxGvS2 zPo|Qzfef!M?$4Kw#1S=?FEi)EJ$^-T^L80X%00DRnl2ik-q0n9Bim){lKm6grTfyt zCjLUrWzK<)l;YF($@CS?YMe(`5tGQwE4?lE%SpSS<9_MCrlS9T>AaS7jdZ1@Oz|Xb znhtudPiZQU!Ry=jD;?wO^E(QpeM3cmfi&K7iWW}YRxn;56St?H+#x5kN%zX+rtGdA z(s^eOZ%1BL(7!`kw-$}8SnKTwZe4)!{aigBn z5w0A0oaql5yS>mqi}hPfzxDz@>+I|8%GT^yJ8kArFQeG2O{TD4*P3y3d=9u>2VO>2vFb)|L2HvE#(tNyzr*sa)fL z@XyR0!_O=>WY+TbdpPwL9=qjbd69}Ah-+1+<73Aqk?aVQ^`v54!t!3|)6Scx=do2s ztZ11$v!UfY&O)OyaUS#d<)_W~pXSYJ?H!6v1c&A`^v#s*qimn{4$Lciaf`>CgXr=k zV%vI1AJ6gK$R%gH=XA)(be8d-y=cbU&G-|8w69J)6)(N36Puu9wQ&p$T~~8(V3I$z zGP%0h!x@J53%y+z&WXX9%j|t)bWdxu0=VVdncVG3t??Zf%JAjhvCCu18=JPmxjt|0 zGB)Xtp38(RfpkySB^dk_L+~?xLCkD#^WMBslrm`@pPJxJ+*VqkE0JKI5#k* z2xuqul#%(%EWgE^H#6sDU<>`~QXZ^RAAFguT$5>^QBJ~*vWa42BfUA2?&cjbh_B7| ztCtmWM5URCKM{><)+s#b)Fqq`wUW7Pvof8&Tz??3LH3)e#gEe_#Y&P-NQh(t_h+7A zGUKPK8c)4zSg_((MuQb;kudL;MuO z+PFxAo*b^5;zqe1U)5t?v?a8AF(w!FS=@|Vp4gm{Fn)Om_anv2gRzuc8o$EBW#<*> zW$zVT$~?Xjt?9igk;~Ip4bT`rVSTlK!ym2T*ZjQv^j39KLsE~5Yf>e~LGzASYD(pq z_0Gd(ucpaCoyBFN#dK1Fe+T8G|AAWZYkpovHd}BHC*l+iUWbmGHcbs{^I%ld6z(FI zotB}_x$LvLp=i=$9#4^Ao@-W4(q_Y4yF1jz{5Cb15I5yX>WWKfjDzdxI4=)o<8m>P zt2|uxucF=IRmogVtYS8ppD?E)S+tct*WMa+d8-OEdEy#1*4=!5gX=zvypLPQLHJ=! z=vJq$oJpNCZq6E-+6{wqk0^JAx*(6w7#858x2o*PdGD}=Y3=9nU3ISPY1;7N~qm4;*D!D>C+ihfa{rfGh<^Kd)buBoKW3yac#PvcOzC2*Qd;fMCX+={stzQ z-{b*ovNmJGI-ih})TrVLied{JJxL67czRdGHOCDuXveI?6U@O?G~aD zhqtU=-84^5#%ED1+0nyuPJT0)k9UQuzWnqZYTbm&YEMw@+PEHL9?zsljeFpIFfN71 z<8ja}r@V<=_wuPZ=3w~8(f=_u;^~g@i5S1QP-65m0_Oe%hmPj@V+^ud)mY?#Lpx=1 z_*n5=8FV0a@5Z!Y=dp0RUAyzaTM@p6cjT7SZ1s8K2AqRTv>6$6r);t}Gg=m(Ff{}3 zI-eyw(jGLW4yaR6H_92K^D-yZj7Back&^D%C1&<1b1;#fOj(DI zs4knFyIG_Ee=%?2J=SI!SwS80uFVJgpjwchZ~2O{9Q{c`nIrwPxehtTm-3R;j-DG;Zd<|G!=Tq%|6^Caoqtuu8SZ zr*Si%zSDYil{#e&sDU`uzABBI=zC*>qge@eOl!L$n&R5J%-To=^ki#gQr~LMJthsV zCP=B}x`V&Vz$qp^=F2IjUaxt5Dj}(jNb@{%-J6)ya)F#aKauO9^WDipJFlxWAGV<# z4it1)$z1nZiDNwBGsc8Jmbc0L6@oF<#v!6rNkj26!N-W%iREzE$i*_SI%(o!={hfo z&m;38{-#T0oGtIROQi3@q^?WkNNOaTj=C>R;!A4#E=%Nk;4(u6G+e&l zOCqmdnX^?K#pRk0{}|age8<{36Zcds%;6bZ7?T?_wlKEN!h3S&*{rm=j<0~u<~7at zC&?QSs!b|FwfqPT*`A^~*_%0;KXm^11(7FVI_(8C#NNYWdW+$y-;>7St}V|wi0x1jx$(=31s+F!_1Y^=7B$CcdA%jKYd3-f-dF=r!<9n4wg?6v80x%ktL z`CA~S{Mqsi*!GLkiODQAtV@W5EOwH^A~8P6Md_`&&r*Hn9f<@k-Pu&~t2$B@sygbotNs3ppm}!lk<#kYL;i}nqGk055BlqZciQzq zza2bY9jf$)s>w@K_ z0e{fFE~_dHhOf&?gH`3eV5knBCky!TUu2YlsbP!eL0eFZdTj;Tvt_H9;mPI+c`)4q0#_sRRpTa zwm9$VgJ#sKOON=2wWa0sYf9Ydyn_4s@{sK?R9IS9dL(E+%J_zCe?zSwR_=6eDyu3g z{MGI?Fa34y<)OM7hQW-FdtX)+sc`sC`!TnJs_H;hwckD%s6l*X2RGd23)T3XyY%{C zCDO8wuBFR*(`UWC*||vPQBID%;kG;NI)A9XuG%gQ`a;KR{W(s>E$Q`ZY2TrLUh|I_ zk^9rjYpR1GyTX683SLdaH`>QC?Dja9THlUHIcr>PjcWCA+SM=AFmo zHG%L|O?}8$QFSDIS5@uYoW0GcG{d!wIFU`bmeawpn!1XhkCE}wDbh-JTefes-NNhb zjkmaRw_S>yMq_J)E3CJp-OZ~&JHDEO2ZR0)&BIT)>3L-g6~3y921>*fKcnwpWz|8` zN=%tG3+ZDH`VAkhKK*U=65&Og<)#-P`0i%5B8*j01b{kq7Y3ULBdPv<+Z{EA|zS-VTA9?Ex zP_XRm+$WcW25YMiBESPjFwb?7&*co&UJZdF+zp&XB|Yx@v#GK3Y?S zRgM^7V@lG)iZyRyIQ`vdH`x33xu!ckbM1ixk=YQQQ{h6#qlGra6k4pkV0lfQKNu~4 z%fj-dO#c`obiG|0Exa+Nup{I{u(EGosFIbb!dF`pWD%;V_MzE_SW*rkl}BUeOsM8C z27R+_YOS|x*M*H)Y!Rd9Zh62|lk*$Q`DuH^3ZtzO@NrCMtH$O z@du3Yyx}wAzR`A8-DA-KaYlP_gX$DrIM8P)S{7|Jc9<6-f|VwQge;>Ii??f*q-d+R zp0!|1_qWAp*&^LLent!dw=dnUQ#6Kz>}86^5OGHg5lc@tXG*%vpKac3h@F+r#0f1m zaooA&7$+mwbDL$3v@qaxZoRTVP5EKRD?p8J-7vQvEhrW!t@g(*Osw{?1}WMe>wsG> z81mQBWK8?(?Rl+48gR%DuPBjuAFUNBW!8mop)Fy*qjsJ9Ah>pI*z<@KcEuOAB;i`d zm>iy00M2o}+@;yb`fT)3%4evDBU5zE;;h6BOxTAiEf4M6kaIwo5yu@Y5R+;YIfLLpX1!oH`Ly&bP-WR35A;2 z0xr$b*D(r@O4ij?`Rme;_>T}(lpYFh+GH+cip+U$u|scyK4PT>!^(WUW_%a>`sWE465 z=0j_-tvAN98q-9S$Ul(JNCVMHZl?Fy?AF{opDV~faQ!DSTq`nCVG0(Adu3KCVs%$y zqzp09dGGo&8Mf=JgmH=4A2l*ge@j~%qX3W+GorCGKQcIB>o=!D=oGa5@0j27=JEj} z&v{Q`rft}nG4tXNROVROEk9`&Rioqk!oG@$INTDD7@4C^GR>^~#u7*6A04MZkZ`vF z_y5@NGa8%(12rLLm`Pv#IYjw$h+-y*?FKaS^LDOF=x9qcq}vdiJ#aiZ<3KN(5QtIW z|F6U~*jSS*cGGWeP9y5&M<%f`7cm7_7~dE<8=uIDVjQ9Gi#9T6AY6%uL!szpPah5$z5MSBhW=!|MHkiR)jZeX&!JGn6ER-ZpF^QJ|1o9_JC~3zqGo>% zh0H$EqjTakp2o^Tp)=?Hp97(}M8^W#^*=ulit$wbM+8H$Tli;-flQF(i`i<6-H)5M zPZ8dgTe#I0y9qFVOM$J{s8b#5Q#;L`ZD%2NtHyB$|5xqEITRbQ;_J@ek8_;2hhs8^ zVMF1L!9J(XJmoRT4zgL{%{}ve&P|TkU!)w_)0@k3n0KOI`f3O5 zY62&7<*$&V8*a;eRpwtD&6dJC|3N>;dv;adU2~*1;15O53(dVV0_Si}l+@aqnt<ja;l=%??0&L(Sn|BXk)sO!dC9N2K11KHv};(wl$ zIVORNc}NYX6An&srs%c)x*B?^uMQtx;tW)UIcj6ee4RAvD9Fg+h)s8>43a{x^B-dG z9V-2YeAQ*gIm{JS+|6Q6Y&qouRl$flZ$4WS^+(d9&8&4BaC7CGW#69ddE0zNySLuG zjgwVoDC*6&+2%F%2ke4174{*2HUHneYv-Q5JGS}mf57CpHuJ1qAmCewm!6N zckF}RyB>6IcW=9Q`>vh7f_wI^u*coo_H5hzz&77KyLav2zirzV>3C(dZ+l?--9^6L z+xG66Yl8z!p{gT>jcWg~ZO(zLC?{L@=4~(HAXv~}chn!I)_HcoUK5OwN#3r#cNJ{& z?Rs$MeA0q6m0{%KlDAk@!;f#C&2 z<^{8^hVznjk)@!*&*{nvUqyXsV37&8*mRS9dE0hu-I?dh+q<>kfc?|fr+)J6yI<;i zk8@Pg2=T}5Z?@M2D(vyzkB;~L^^4Db>cyrvjrTtBVpI2zKm57$iI&f?JM`j{pXO%m zj~d@lRb7GBU=1-1rgm*rHREBfe%RQ`KYQZi&%F5NCthrM`^4w}`o+KctZC`VkB|Su z`!gm!{^^PC$6xGz(k|S+?Vjxg1-`rU_wKyk{&Ca0%=69<+uMo@x9u+S?J3w*w1-qu zxb)V&MY{^O@ATcZH}BqUMNF`*cSFZ&3A455{_Tan?Ym;nPnxrnoS~F$)#WwyW>4B* zVV73h{<=Dh{;^69c{(&${MAA1*g8{nojI!{JDnq$&Nyl@oUe?Y^At>- zzw}7VQO}^W(2L_~A9Eq-LYWXBA2a_2&3|R$?&i1xxhm90+pI6r$TkzzNKNM|o!BXR{U1mv<=wczkT0bXw@i>AUAr3lA zC+CJE%M*Mu1l7EZd3N|{?47Zrv>2^YQCG^*VW;z$g;E^pcgW=w1#yD`Wp10)?=cq2HY9oaMrNNL<3q%>N z&H}|WT5uZ~Ylnske_8z@Uu}Ss#}VQje@4mI6~|-lkCjp5 zQ>w{?msiFfzjjA9CPOYhUqwy1&nJb)3s6em9?TT$%MLL^TvPTi{yk^d!!Pfy^p_t_ zuc=)tnRbrx^DF!fM)v*&Hs4K!o3}8Z=rn{QS$Mptp4k~~F=TFInsi@B+l(|+c-*&V z`)jtPJLT8Xj&mz9HGM@76>dwfDRWNMUk`CL)rWjVczveuf_HZrSI>EObi*CK;Bii2 z9YKG~%4<1)6EV(m>FzFiuWQb!px4Ymg;X=U(~N1kFF+IrdPF9)$cc?b?U^v>n^>%33U~ z4Z;IHtV5Aq6|{HmEhxA~9xSb^M(1O>s^Y{sBj>Cp5qa3<0e@+AeXU)|svYpxU1Jwj z(gy#`$0b;n_^biSResnxl4LyVI)61M)~R(k$eLAQ*ZH~Td~_WPKW2f|3NuwxUxx&3 z;sJ*&v>8bf{?v!eBEXThb%)kvn6^uuBfoBIWq!A|`FH|7mmQ}-4Z;sp9roKdFu^%F zTU~L(+I9B!gZA;7db_mFZ9GK}8ZDU1^8Pn@H2F}Me&Y)wR(Q8zE zt}XN6V10Etdh32#h0}il_wYPg>nO8vSxfl{gFi-%!|hboRLpCs3?t2FJp2f{g36|0 zr!*hpFUVQ2Iz|12okcr$#CQJ~#0r zML0X=gFOC*a_18}I3wIQR2`{5Vwcq$n`&lib!eSava&Q-X@|_By=Vm!5yp(Lqqpdm z@eiHpfG!8GSuLERU$<_Z9XwoBTgz)#msy!t&oRF#w(4QOAG;DTpDL>JVR=o3f8FXme!pEA3e^TTtzREuWt~~Aa6VZ)g6pbk*88j1 z6XWw)sP(k%Oxksop(6n!zA$$Yq1tO|&753}*;otg=uZVs0P9J*5ejB_j#Xoc4;p>D zMsk>C)yI$2)Caf3$l2tksJHfvNkHTgs;j84UF_8v-4nLrCmggfzVlWGhc;eD=&Gtl zyV=7xA5lW3&R#h3sVT<7cp_Eh>*plnP99E;auCoA#T>si%mw2K2hBQgR#Gr!MmUiQ zXg+qt3?)*!Ioh7Husg%RlnO$&L%UhQj1h6>E&PS&Z^+PTEI|6BYaUyU07vla!gA;A z+}fBqwwQp;@)}t(**3$S+F&e zDJfaKnzqBPkh375D2`4%LJV(|otvl(@jwFyWA0f?6$7TcT$ zUu)bP^I@quq8;hF>qxX}IOc0$Xq{K<@Z;uOP1lz^%<_GR@vJt!qw8dv;M@3=veyaX z9M`3$8rCr*VXm=>&PVl}b;|KS>_XRRFqXnlC^{LcE3FP1 zi;D@24naiZsEDtuzAC`X4>v^vPT*4?bSIo^u-tA;;|`PJzB6GphRH0PbJB@uQgEFk ziGwxR*JDPZA}^d4@ayoQ=2EenSU&kdon-a)>Z6>wAthZEVpTS49|`abTinrNE8|gP zX5Ha-We~ApLP7g6x$(wuS-DZn0Gt2BK4I@P&II-MoBwEW{i3rT-;_n0oNw5lF`QzF zollm!yq_t>2svleT@$2sgmGi3=jIf%H*>=+8?$b`?e^?DWX;}nYe<)z;RhOn?=$ul z!)FMouQutFa^u>vBZKG1h+HFg&yBZZQ;lJSA{H7xO}<4LP1l-MuC?a~T)b|W92_lN zB8XYf+^j+hhgdkw$CXR@m@#b{dwI68GR{Uh5)3jAA{yh2k2?fd9J@5&9O{gLWx1AH^&LoU3Zf5bXsjFf;XP#A$n3RZ0#WWYyrEpa2 zG#l~l97SD*;fPkw(c^@dCd@`$Zde{x(a7yQH(()u(PkW7GoyK!^c>c$X0E*zQiOSp zSaQ>g?Icc&h)ld66C)kNb$3!F&S#ESDmkY~V>FzPuN(Ut*%MrYi|3AmbL`)w515Po zBeljjn5fvzKR_K@5s`S0R}mj_@zQ7v3i(Jqx&0-}Mq4uzE39faN1w*@5$VivV2KS- z%`z4_yhtLRx_U!vR3XiFPDCe+F^P?b=9D!WHW_<3ZwXy&z~&c^>R~vDF15|Dz;rmu zxyCMVVj%Jgtl~FR*VhI8H_RDuCj}7o@l1ll(LmG2&2w!y%eNV~70`pdHh&4iUO`G= zJ|_{CNAfL^jn40bqvCM;eegmKId1H;Cb-`UBl<8%78=|;(o62p)4z?miW!9}OZ5-l51lJsy# zIbtN|*#YNe4Y53C$_?g@Rn{P#Xs?x=Dk$@tu`#Y9Gr`$PI##-r!?1jxCOzcHIYJq$ zBQqOYO>7x?=IF#y^Mpmj>BgjiTwl}mL+5;{#U@vz(h(fOB`U4KnXD!q8PPPyH3`q2 zGgpoUxGA`EA0!=U)Y!sos+Fk8T zl~R+6OVrf{U)MItui&aaKUcQO!*w`|sTT8%_?Zn)<>JhS$kM_n5AcEVG| zwWtn{j(5xUUS*Zp*(7o)b3}xZjL}3pHi{>bGwwNu0&s4lZD2d1H|}E2LKhunc7)tT zVUZzTq*N^bRn_ds#*j0o8j(Tc^`dHQCXCUGIel>KzA?9cf-nZlOv9U*ICZuudh^`$ zF?0A2>Cm*dc!wBpr?YeEg6PGpVChD#o6SJ+@nJS?T#LPVP1PT1V@I}!oB4DuGUPEs7wP{`uxv!VyqM((F-0Au$XXDyPB<1k zoa&gnkz0fl$3B@@mc13DwZ<8?SyW9b3op{y4U9Uti)q!u7Rb($ zh%@1Cgq&zCW2lVG`NLPtT5|;9xUm}x9xY+I?&eq;4F~MVlHeYzAiWTiaG#r9Y~h;y zGZUv9zE;G_f;v&TlaMj1QwSkNnTf_S1dQzZyPH%#C;LGb(pU^UA`>PXhnR`%yfN)m zIDax`v6jM{cFqp{l36Je3oo9Tz?X>G3o?8c9=~^c!5*XLKG)v5Idxpm#e5e$7N4`F zQ_hvC4zUr7=}^BI&GRbx%o!o~@aST-olRlVAM85{5yue@y_}f=4DTG{@*%iIb(UrA zpxcTwwy_abj6XT&{KE#@O0KgLo+@)*mpPJSoKM8$c)rG2t@H=*mm_Kx9cNA@{C9Mm zzE2LUC0j3MnGecitd5b4!kl`WIM#RqKF1X>RMjwaBrc3VqMOk3-9%B58iEZl;t4~TU3*=;OJ3popVm1$XNCK(%?O8qNAf6R=9afO_>u* zuLF(I&p@YrMTPeiY`u4nuVB~CdwqNM-gS4u);)Wo9fX^I!2Cfr=-z8QDi%ebbJAe0 zcjqWXRJ5`4chjcWJ0nwf@|uk`Y#U21M^^pwj6rOLO`F1TMYQJp1dtPSnAngUm){9C z(~s0(c}X@K!|QmkCLOod$-&-Zl=ND;Hp9)%ktHR4wun;9<%1(uFKWTa1NrMXE5i<~ zb2h?WDxY;`8|Fa6*_VJb>O{jDvl218=Ju!}nF)7i3vV!zfp^i!YhL+_@aY~GK%&9IPnJ07dY5RuT&p1E7 zbC2(i@Qkmi+rDRmZyP_dNWnUgfxxMh7qdPZpYB9^4T8uV#n>#$)wk_!7 zb~@Q}8~-iQrU?eNFViHIm+MuQX<~`q4RhOHvvUyY zIuzVqZsbP|aND+{{4PPaE43B37(49IrdPnXqhM#@Ju|lDJy~Jq8FPnuqpW9F(pxs- zs4Vv35?G7s)|{#?Sv=CsVha13ol92xpEAfRCh?CPk7Wj9!=A-HLbJqBEM9n-fmpoK z|Da)534wp)AUI!%VE)qa*+*l;?Pq+fXz`I)q~LOG-*S7Q@st<;sLaoOdE?Is~cNRz}yi_jT(f^2Xbg5*pmgMh`Cv)fh zdjm@iQ+Z&U(X*JfKi^TlW!LhfPKnp7obo?d$Ce`>XT@E;{~KsoT9B9&i(ljC^7T5- zulW*CbI9Ic#;7ZT*j*)k~|Q>%-4mM50n$RNG#&^S^=M<>qnN3}uEF!5BM) z+MP3n^E7giC}_z@U^z0w<|fm}5_^tP`-5p(daR9>nxC`#W()bhW2^aORh~aszAzqi zLn60LW3SB5a%c|iX4oMzw# z`lAkMPQ%WbQCB=G{+}Hgl-~JM1S_s%3cB@{W5sFrfGK~T`H!FFF#XZx+Fav|%W|b6 z;}@&0f0PEySEe&G-}qMMXE}8BAZ(~st`5t#8|nAV_ML5`eQPr8HO@e|WG?%^JHF?x zQGcp>5q_4#<@F_^31;*ifv+*l8^787EQiatzcPJXSt*Csd2@NGdB5|x}6 zcwJkL)^kv^Gn}RQ=Fy0sB9VnJe0zr7TW+0?yg#>biP9F^|GA9Ta(WVZKS$S`y1Dvq z;%M`;9IkL&%zztlE;oLWa*K||9En`G)GhCKMCNVGJT)Rc2mfNOS4S|B_Ljq#{+xPC z`D+WG|EcV4M35^LxZyIYbxVprG9%^;(?#KdSPo^ZvNdFQ&QB# zS%ifjm?CzGzP=@Pdwv_dm~l_P zy2}oAiZUNRPdM5jsh9A*&lD(mzZUyCky#8v?bUH&Z{$n1>7*jh)c{lOTz^da(}k<0 zV*}sW`0CZ-xsCR~$vgQ5T5x(Z-ZY~CeegUPSvyK%vaJtDc_{=FiFU?w)a1g!MI6PQsdauOUW6KMmp{UTl9 z)WahE;7B#!9S`Q#(I41(l<(BtvswyYE7Az|gDqgeo9U-$wWPm|e!zlvh@1eM!Dg`J zDfj~W-UDA?Pdj{pE$@XdF#Ut@1qPlaj^0bTkBg*$EuZ8&eZi()k!*1K^CEd*UO)YS ziC;#3;LHI0fXQE_U-SHH^b3v;(Jwgq4f+MUzDd7e=C|kfj{J=N&GVm&1i-A9_=a1f*Ya(rpnU`N!JG^< zwV3aZ1(U#xJC$^Tc{$VvC$}m&4UXg~83MEKRWb^8-$#9eJC)2lv|0xDC`o+HYG_wt zg9Wcqk_R^Y1@+8*nUXPZ0-Q8hp~SNfehyI{?5R|e1vVU3QV3=Rl$3x;^-5~Nw4+KI z!PEx$1lwM#qyrpmgibK=ap(jKnv@KK4R2O54raVn$uu|%diV4G2_-4uFqjTbJ*gxI zOniru{b1$0lvIKhA485{?Pri5xc@6k2EpRzX%`H9AHKoH7isqZ?fe^jfR!`w0p|ZF z{DUP*OA(ml(Gmj1tECB?P0-Q?cCF^i?7@!nwDf|l7ibv(i!RnOZtj!$0&8&OGA+q} zLAfimWPsCGX~_dmU!!F|7)a9+0F%~hIRSbzwKRi;w`%DFXEtdW1Y5Ug83l)5rDYP_ zpG*5-Vji~12Osx9FF0|pmI{OU&<)mZ*K!i<*a6*O@~fd6Oxz9K;0QPX_CG+oV8v^+ zB$c3NC0f$J@lq|>VBsMxg{9CNpnWi_MoR;D`Vr^_i-NQZ4m=868qTDrmU7A^f?#yhnPgD2jlWeoJZSIab* z{XQ*HzFNjQv?PLs&uB>jLm$zS4z@lE-JpCzOFlUBcUp?U+)rt#1iPQ3e{l5kT3W%B zFVHVI2=;^7UqTN2ExW9L(vk&^3^G1o#@CS}Sn^FRr@+i_BS&!N`&x#;qL&zdFn1a{ z!Ia->@%S0v-=PO!#eX4Ja2m`4Psdr34>q1}$$oI+0!sp5!Nry|f=$Vmw1B6su%rW= zz1orcf_Ffg^R6^qTvSB?DmcQA(AOnlsu9IzEE08`#%NeMX8WJw#?^H$^sdfrC6 z;KY-bOo97bESUvQz1@y5EGY#0Ux1zfdOwW(z_D*a7nu2NOGd%KcP*JT&&S{o9Q?5*iARvzPb^6V zXMaX};PfQz8T>i&0#E$Hl3K9;mzH#b#WR-lft~+p$q?A_vL$KN@PiAL3%195q!S!W z;0waRfz`NMHH^!p9&82r@(Pcf08d`WSBHb$X&z|@N3QqCDf9jYkMx0K>pU_9cHHKX z%v$Pi@kk!no9B@Vuw^ITN)9GJz!#u{Cm!T0(80ld^b2N|(eES3zuY4>IPfs-nENBp z33k+aq`+XEM@qmz$Ri=J=a@%Mf)&R-(h8$A`YrhmdCJz&Epkqg-JDUS@B_kA9j0Q)}Yk!f(I-ylh7UHSu)e@K5|-#C1L z4L|kB6gcy1_^D?+e#1C`Cw|8`fVID89Kd`lPIAHGRdG@XmL$YU2{?9PoYaCd7sp8> zn0Q&7w18<>#7PI3ks2r6V57~q+=K1c#>o(vdR?51fekz3WELFR6DNsBdH+xxzK5|h zagq(n;W)_$CjxO&43^Z!NdWA6g#JKJFiu*{^J8&x8a(~lI2i$(-$c1%wEsk$G=rH> zLJwHc5{J$}cWWHB3H#no|K=V{YQPWsK%5kTwO!N)b9&(iJo(RY=ooh9`8WxIvtN&s zli<(`^bd}HlXk(pf2Dsg_d9Vi0ZxK5;NbV-WdCcC-;a?O*!{CO=>%JT6(^M^(1%~g z$q6tt1OI;s-M@{KEO7jHagqnl{3rZ_r~ZrbU>CdX4PcR8B~9RjwMyE-v{kF54-Bnd zB|~6s;wl*fGcH&qQ()zVtHkp<>R-G{lELQWRZ=?06|#LF-^kRC7NU|UAKOoP*z@#1+s_qWGOGFVv{FKJ-U1M!jt=J?_z z4?Go!mm)B&j%>#p(BmNG!L;M?QVeFkHePDMq(|fBBv|yico_hb-xe<;;PBhyWfJUc zjhC*+>F=HK(g!BC#mf+Qx*dAJ{tw2B_ayrFOuVFkDIbcLbg&Z421h>{FZtl$$LSZe zd*~OO_$2*;eV>k(b};X`cwtHm&Y`@PdqhQWXuS|lIySy?BHt+FD zViV(h&?~86`w6dv!1T9zWg7H6L4R+?-aYA+7O>&nUKs;x-{%$YTj2k*)C1F=^GYU| z^La16AniTxl@hS~`|t@i{{TM0#7VF0e=GBE#w!7^@@3wGBPj{e3l?9QAcLTNO@fSq zMK>gfr!s}f^>n?n-ZiC%*mlWa5OhT zyic%RY)g<-Fejh(K+k;%k^@eFg<#kH2~q-%fwf?BL4q`b4LcL01+08^f}8@|!9Fmj zC_#q6z}^HI2PYm#kXdl@p#*f4d9g2nSekYN+&>9FwF#02_S7dx7FhCFf)s(NO$kx~ zj=v*85?dI*cIX64-UprF$omr{-`sxyzCllCf&{=R@C4ZVOoB9n86QfJc5nhb1vY#b zI>A5}bb{$0Nsvjf8=M8RKT3aZN3PE%hz%z7BuFNh_(|k!uphaA?O%XCu=R9;G=O7Z z6PQ1Myuqyh!}x)1|BSrB%&#WM7-)YjL8igR7vTRLy#E*Y24}tl-(b=h;|JD`Gk##x zM1u5yjgtv749>hneXwN;J@{+J{g<=@wt;D2(eIE0=v{>@z?np7Zbe^`kSVx78Jh(r zUWV+{0x0UXSQr*~2IF4_n2 z);X6jYt3Fr-hk9b6U%f(oTw&pTA5y1Jxo5UHBrthCMlM6u~-!si#lCgEnl2I5 zMUHCl|Gs|r#@caz8~e5F{euOv_QO3~vsA^0_-^R6L3 zbd6X^*HZUd>Vk>aiAui?I>FxS#G2yTP7__5MxJhssMPC47hNyXaJ{GjFy{tQ`)?pm zbA#v+FlDW%T$7JmD?0l|_`Fe6(oLjSZW2`qwu6&5i4J8TuMF}!;3$}Nv#5fb;dh;= zmUYxyFUpe%JsY5hyk5lyvD(1?jiP5a!q2T@oxD|K=2qlNzOV0g(T&-n+Oz2!^xgq& zcOc6 zR?(ez(RQAwH1dT-+u&!Lc=GOLT=GQ?<%=hU1Zma|@dOIVgB6MzD->P$YVj2K7$b6n zqdw6sB_cy5&{xXXm5Q1IPnL;lD-&X0(E|se`;e%@L&y~D1INH=Fr`vdcBKe^E?PB# zqm`mlt3;Pn(eGh+KaBp6(w_)0W=BM29g$TzVDS;;SB(xHU|v{h7~6&-x1sG@hm*E>af-i6-03;BT! z;PAUd54{^6pAuF3lvvGAK|j}nVC#Em=RNQN+E3H>)9?W1wu{x?&RCI`Eq<7sGeh_{>D0r&tf&#t^Bm*+9QFE;)8`nQ&x;x+2W|JmQ$PCnkI4KB zB9&jDjW43FUqrV~V^6-sxO@rud`Ya+UlJ`}hK8?*mHQRZQ(r+A|17%epP_RQJs#x# ztH|-I$p3ljJbQK!jcCk=^C{5tdg>+k}OfV1H63-JDe$iT3u(P8rB-(;M= z1x^2gJsM$Nd>c7`TRbOUWE@_kubUy0TF zE85~Z_1B_HehqEEhPMAej=y29|3+jQ^v=NVjHuQb==d#Ve#^7pVRL>bsu3)nMK5O2 zr{6PHzh`WIFS?n$I@4M))nrgBonv8wFHkD~0`k2VDmBA;*PNtOXOfbJi}P!=p4M zxoecpx?V}^_4EzSfT`s9r_+`4t|jljR;k1r$#dVRbmmRu`)^h%?PjHm*3oySQjM9= znW;R}8_8+ks&v7v+~2Cy6zIJjS=^2+z!@+vTd9-Tyw6s;;SQynHzAihm8Wa7l8h}% zk8h#OtCXjAt5Ty|sehN!ZFfOGSadgK??r~}C8Xsmm7TAo0&D=A?^CMxKKcP?z~=jr z-~G^Aprofj=^3zl2Qu729)G7&>Fhu32T$#U_g&Dq3z>sWVEU_-9(y(W61(AZw_<-v zsdjK0OkuwwZ;w*NV9{QsPVA+f2cf%|vJWXKen_cWu>B#W_p_IAVjpGqD>=DesV;DQ zzp|1JFm?y13l1Gn))YAU7fKiVpw$Q8pqCAR#u6o`N|YJ~`$~~jnbHGglr2}fq5>WA zD;e=CEA=3I90!%|XHTTPQt6B;r7EkG^n+vI@WaUDVfZ+#)WBgdz!klERPh z{3A*VKB`p3N8#_I@C)w$n6kP)hAy5`D({q%z~3U*zh!(r!Myo9+Wk9d2d8_u|0HyL zQmGI)`$=^AQ;g528QV{Ty~>*DRWk4yrDi{)#QrS$_*rxvYzF(mF)-~prE;EQ{`DzO zOFwn{X$vg;M`bnsqte}9pxhUfWPTAkzDPT#d3PFJ`x5vibP#L@8^5en+n15Smz9=pd_zn7g0a?7jT>B<-?VE}7E}T;~yyH`62!NQ0Xpk^hb>0kC4%e z%9A^xRM7-;Z9?hUe`8+$jJfi2^yKGChhC!2l+yXXL`J_5ZMIby^~O zZi!mWf+sK3YV1PxaFVpjO43q$kycF?Y26QwUBq6_#aiXF_jdwp2gkq}FzphpaxMWc z)q3<&_RcQTdXnGMmuqRiQtP&>v^C70VD8mgHD0at?A4Tgg;oWxV1MluT2C2#B@EeG zkJ#*;U8Ak3YqUzbmSYQPTDPRJf0qUgV9^cQYPmtH&KtD0)7ke**Qyq5PuDtmE&E<; zwP)fc_VI4ko@DkJr!%#lzJ>k8EUm_~*zdYktIS*32fIz{{A~Kl);jMFe@<~ zyO8-^v~ic#LwD2fHmy!?(|W+{W#(%=zFm7#chc`p#-WgLC}jM0YgN2k>-OE0*`rlx z5BsFxAV?BZ*+p7+fqg|=KCLTD;JZYtZg8|j>!MPv8cNyUtAOSTtwt-@<2(dS zhbUXgcvhkhRa&)Hu{Q`#gM|;HUk^hQm~@!(hqcvrSnHgCRweAMwu9YZ;t{RA)#zCb zvSP2br&e3?2=yM(DzT2St)m}s9Q3mHnibSm1=t$YIyr<+gtRWMr~WamFG4cHkHv4I z{GN>{bBBLriOwPqaN2{93r@v}%3Os%xC(y|Bo(Vjui{jTO7@B__Tuj)5NjsjbAhCo zB>g;5Ip>L_B@*8xa^?c;1hdX3wmzS_7l=x^fSBMy(c>2qOD7S((7)#*+Jvxtu-FJC8mc#U{GH;GEU3BU9v@${~P-}R!~Gez1qiaNEC zK0(RC-_L^n+ldoy$M*ySU=DG5h9o~MXdeA;f=)OZCgYSy^3~n zp)D7gfVo?d*;bL0cY$|_41u%Y*xmTycO#2Du^RH=4b0g_AKMt0Z4BEzJlhV<_aVpo zh%G?x4(@j#j~$|$b|UXx?20m&v@*IHqdiGbSuB1zrbJj!6SKrA<$ET?_UDlrR0Z7 z86$E5C;i+Xq|JlGiR1wGA0l=*BvxuA?<J! z2Sir{L`)u_=m>G=5&EwcHCZdV?-Ar%C#r+^KAqUUH;9aa#MmKdB@oD{=lxOQjibal z4a7MO@PC{b<2aMxwT$6w(F<@8o;M4sdWM!}Xhif(?BNN1Div^P`#E#jGN zrrZYp5*#T@=Pu8*&C*HvvX%#)&%2*Q97rc``-!0bcyD8g7n{Ck6 zCTbM)J|(*GDQI{~R6jWK9%S?$(PPB$Gf$%%?ZmO~Wjx*s-{1f^3f8_)RLlF2*ZaiE zA*S#98^-)^h|N0SwS)Mr1D$$5<3en2e~|I&6sx-vxjn<&eMV#kO#2YI1LF8zaQs7J zC4ZRlCZ0bHHgqxPK7uZNjJbPCP%hDV-6F%?q9#E3J8a0`iHw3%V9wve_unJa9@-^d zuk1kwh}UyJ$=HK!;2@a#Dbbz8?Y*C(PcZS*$n~?#q0iD6@p)n&Hl~m9|D0HppX2@K zMKb!4SwA+eA3XtczQ`Q=BKe0eiuRr+jy;WTfG3I7XTgjw(eIZ;LSQSH`epj$_ryQJ z#y^QQ0(K4{hXK)nuOO2lWH*F;AQo@y$P?@t z7M=4=Z00xN0qg{uza@I|TO#cv*pw0G;CI9-{En!K?=Ys{VZ2AFKZ=fxV#h$wcd>=v z6)6A%;PiK)^LyCA@1slK7i<3*aveiIe!yJ+A@cbVI{qWED#6}y@ihEIbnj1@7sUI- z`MT*B$bjEY;`!_;bZ<&@%M|+a@1iRI9sa?7a2m`bwjcZ@y#JClU(xn�vce zz5We$fEa%NZ$)N)D^|ttum`_GZnNma?-_&N!#CLTvRG3uV_yW{6;s(I_^jY8_aTMP zs_|Jh{-TACYbiz$TZQi#PwWw|tRye7ix*!TOi92WOu%mhJHe9G z`2MSjV^=H7K2Pb6^Qdq8MFFAvvi5)@a=C> z(s>iK^P8Rle;G=3fhURQ+lc2!Zzcz@j{LwnWB?|vXZcu9?6Y3!DbPclpGmx54AySI zf8Icx1&)A)wDi<6g4-mS8 znDY+$1ru|W?#>~e$Wi!n^s`y1#?8nE90eO+rF2>@u~;s#7&s12gE{023b!gbb(d28 zcR^>KitoxJCfla;G{1xQD3N=Wm3}X>xKCN__bHunKm3s=s03T@Ctp$kk}F8vL7g4M zHanG7xDy_C5~F~!3z~N+83bp*%tGb$zM7@_)$mTvpf#K$NF+DVN{*lxY#~R`xffYK zfUF-Nj(UK8imCe=^y4+kle!-oi2aLv$}`|sYRnHW#PYe7&`vC$Q$^fTr93A>%HuhK z{+vL5k11V3d>(oX{RFdLr&IxPdLi*SvAXnw!>?CX67hN38yJr_FrMc6amqZd^Z+>h zI5Fu-boh-*%bS!Qc@ypZm9obEiuRk(6Jql|V)F?w{jKO9F?tAWd_s9nz5|-y2@UTe z-g-BEzFUdiMr_u`I6p<)^c3Uz6uAL|@1eb?(Ni}*m%Mgmh1$^rF#EkqPP|v?e(==$ z(D(N-{vAr$9h3*B!PfU9xA!Zl_yGF*0q6pUz~T=oa=q}-sjS>iB}vZ^lRbm%h~9dskIQsK(+UbVhZpQi(N+o{+B!{0jQ}6{S=D4?O-K<}o<&&&Y64>Gnb7`BldKs|+dF1x|vV=V|+S z<_|apCV!3luPL2GJfBM(pZE=AKrBx@uUf$YF!2SY123Rk#P`GC4461fyTtkJ#Q6i? zR94ovkm_FtH{|AKr#&j@@I|96Zi83f0`fqzxH<=f0f@&G5k13#dQDqYBL z3%|po&NoX2bkUJRsH9Y;Ay8prW{Rj2}90k3G=d!I9P4$|ir%L>__Mfyz2x>%jTMj_1=h*h4O1 z;6knY$R!MuOBf=DFhLF>jhunWA#{ShU;(*`nOs3KF+X{PjLV3fFDK3? zhtPHfG4vJ0x?u8^&~+v8A~<}dwmjq&vaZs)2i%{kHTeZ;BbSh3Yq7~4RFE^6CU@Xn z!~ONdpVvb#m~;c>$s@FbX*bf(jaqxi8}wvAcLuz!hv)TLomj8+%zEl(LO1z@5cz~j zu00!|Zv%1YE!^LtIfJ6D>5b4wj-fA$SeG2b$y>Fj=uVx`Ob(&3^Glv!e?D<=KKT|f={~Ir@8c{p zm~ub-k`EXr7ZBL1tqyVl6MMCq1+yQ}R^S1xyT}WqK18fcKA`s@ZB2qn`?W404?r$J zodo;$LqA9!z)CEkjZ*RjWzbOu9bj9Twg$l&@&H+W&Kdg|H?YgEtx-@a;klAnpPWDl z?5@<-IEd0&S;YFaU?52W@9)XAXvA=bqPRl7`{0UG@a@5Ia0blZC~E&k{PSBy zHQvfl+$NssO(JP`62IL^3Nyf6>K1m zZ`lfsTZudGB1XAOtYNV5Zc(*&Bg=d7{qLn7F?-f_WO$!w6Sw!@C)Ny@ct3Fm@jIW@ z(J9344ZB2Vz8YB)t558~_b%doFSP9yPiHY>@{lMKn@>F?dSoACc0fGrrwjY&p41xX ztP$lUHXf;kmPhF45&Zf(_^1<4dcAOdQq*ZM=cuS6;^Km1(0NRB=P~LM8_yi4%wLMG zeH}4LBQhn%&3=PO8#n@biE;BEhfc5=96l+Y$~U7UZx%HSX1oR2zJ+)Zl(&k`Z)R+W zbqB!ICm6dYkTIC}Hrjm~av|=`CjRXJd%^Y=;-MDenYW`GZ$}5f88Gw?X^S2_k;7Ks~S>*dH zbUcgfz?@Hr9{)r%o^2+EC3aQjGfO4JtdpN&e27&mKMg%#pqJR_GmOt?#7ZZ2Eg*(1 z0jI#k=Xn2|sP^a3DPr0*;@P&(i5mKxXz%Bt`SZkdpC|U>djCI)KJf+U{vyx52pz<- zMW-qAB{zo6_%gcxW#0c2?T6#nF|c@m9MAycMGTwq6~^VOjK^1rOTk`{y=Oi2JhA!n z(Dyak{2IJ{owmLq(*F%&J}`-xHvI+eiDf5=X%oLCmQDORLHyeLFVIB{+crY{2d4fj z^YCAhEjS8Jfob1Hrr(x?6W?asiEn#}bBS+tD{*azIJV+@=rXbFX>dQWYzr}M&-WQe zuy~Ad8AA_2&kw}PA--+?fr$JFK7NEA|A@K6^*A^_F1qVQbmzySlYfHE`-w;k*aP-V zFb94rlJ{?7HT)ZT`7`wAXV45LPeR8e_JI9w&(B2{{Tw~~1^WGx=knY-+7r~XDffmaY4y#l}MmG~L|4|{I|7e{sd|If@01Qm5_R$Uc!)vc~N z>Lyv^Mz^|2HcAxZ8cX%y7Xj4lS zEw$99mA2GUn<^?*qW{mG^I4uks(tvq{!e@La`rvv+@~}5X=ZjdN9rlK##=CWlCHI# ztnbqn)>G03^>p)3*c<%``}RL!FN4orka%`tgt1SG|5-OWA$@OPzx0M~c7K4q74|Ki zlNjo|bff4a>{Ic~V-Rv+0q);pU*bu_8kJ@^JlK!4;CV(jWGD6_{n(RCU~iI(JxMe6 zB$53LN9BHodS^-B2I<3IqzKP8@a)3TfqlnL>_70_LaSehJq~2>V8fAzy~Z}|Id($& z@Z6&8P-8(i_8FttZ@93(Xgkc%dk!=9#a=+$u^8noHq0=dAy{~RfM*D19oF$~-1pzU z+%TON;N74L45JJ8^oOFj=Z*XH-8qI?bg^Of=o*E(FUyTkUMaLVxqw?1C553^JT30UpCC{CiwRb>LwscN=DY z3-W7${Z>QI+G?2TU&DUrYlc4fHN)t>55Bq2FzX+}{-6uzrOPne9>eD!;PVfR6yM{9 z9)8>~%bvnI-;aGnzmc-zX_Wu8VeWVq@>#>o8^m5~&@d)n#vbk!wEtIz-uo+Kzlp!$ z-0n1t$heW3H;(i84)&Yxz<2K&slIm&v*mrm7@ssSfE+2KsdyhI&0(bNgZEg{@nwpA z9Y)cPEhn|CXlg#}c#&)N}=>L=h&%7LZ?g4nOXQ4xHU5NLG4s_^)2g2qchweVe zVGJFFJPvjk9d3u-?{?t3{0_b35Qnkt5WIVGs6+Q3ig$w`%OFP%bLbO?!6w7usLXI^ z{TU8D?Qn;YdpM3A;m`v|IJ6STo+BJa_(;^{NW9PFL4F>*cLO;F={(w@hmXd4E=M~Y z)-et}a*V^MKL+mwc^!Jli}zr>4x{^6@dtF@W&FocN0K<0lfDE*#$Xxszc9M>M-(_I`rnHc)#cj`0)(9 z_k}O9w1rS6Wcz23_n8j8<4oiW89fW{BdvCr-B+UBS2}bphH_$#y?Zt~jOGf5Sy$yS z>+!BsevQM?SA%zmu5%dvTD-ec2iq?=%-XM@EWGoSc9%o<-Q{pJ--Y+VwmS5>tq#re zRrvX))wDSr+uIyw*SBzfzKuF{ICReg=ra$3 zzXQK?I`qCy^q23VZ{XdhX2?#+9gt~VIA>ir-;fiK?nls;M;yi&r0-GKKk9JQKZ^Iw zAhqu~bl3OLr|`mTdk@+Qxf9a+Lx-OALmdC1!_kY+!}x4H?$C=K$1%vXCmh=F6ONSh z|3&}&F>HT~wm#)>gr9Pl8T}4ppdWsJ2L0t3hc^B!%Gic}^Bmgn9NPMvBee~306YTe ze%_%6pU3-gkj;>hpE?|MKSke!EZUCu^0qr1IX^?c`Wfnr_p<6=!1;T@VT|H)@6S=j z&*9S-9eVghl<^Y!%}eml%MQoT%MRUq1?9i$FdBwo`x~8dktG`thK!=v33;TZ;5cO=I#jq+Mnj<{5bJDTIqD!qn?m;GLt@rXF2wYCY$gDWm6`TJQptc>&&aikf;; z6!Jn-ue%WTYfZg!t?B4mi#D%C-Lmn1QMQ@l!TVm3T+?XEHMPu3O+EioyjOG?^1KZ1 zJ6&$3xMQ%3q3!v253A7B+wdM(yvR&x-(YHOrKa9f3ckv8bY5ke4P~$?H}&>%^$wbe z@0F@|%S?T!67O_fgL+(pcby;yA>CD`?ythTQ`M%UzZzvgT6lM?KW;jPAoDhvdVG^< z^lifXTANM1VKd(MsxkH98dLLaG4cEf=jJ*y#k?NxOI>do9=t!+{&`dHgWOqX8l5+o zdjAck=BYPzyi4XNsW*+C^`@DR_sN=XMEh?twYr;3!{1=)nfOveFJkCs6Yoo#M%pbn zABa7-nnt+MG`2OG=1>#*#_c$-x8uF7W>fEKM*HtZntMz~>peK6|n ze|pozy&k-0_BZ(NZ#chyH&cq%I)m5_turalVU`L5|0dl&V64}QWsW~~!w z%RkUJ|6v-@_f5_G0J=$2&zgiUccG8&GR++(-Z4u_arC9&9kA3CqcJr_8;0~ONYTRk zq?k2$_iT9o6vJ}johoOFk^d>YD|JAMQH$7%IF5JHybDwG$iftS7d}NFf~{T{-aT^qf@jR$UextW02>kQw#^T)e8}I#7`k&aZiS%2a+C0dLZe6qz957NO~aY zfuskL9!Pp1>4BsNk{(EUAnAdm2a+C0dLZe6qz957NO~aYfuskL9!Pp1>4BsNk{(EU zAnAdm2a+C0dLZe6qz957NO~aYfuskL9!Pp1>4BsNk{(EUAnAdm2a+C0dLZe6|BriM zsAV6t(5dxNN^_lcc-KrFuTyW5D~Wf|Y$-3FIqOo$sgw&S(~P##Ko7-a?}ezpd|S$72Gk(3_F zqbQH2JciOsiJ!GWaMm45c^svW@_0)8+zo=W?nFw9@+8WWDNmuq&)^_9>lRTird&c9 zpgfgwDdlODLCR&6r&FFm8KT5b;~+Tk^R^<-q6||mr(8j~k}^WMit=pAb0{+@S5ux# zc^+jJ<@uCrC@-LlQeH^8mNJ_Xzj1`%thb15&Syo~a4$~?*|D6gc9QRY(? zP_Cmaq%5K=rd&_CfwF|Ml=3Rdt0~JUH&T{UR!~+_UPD<$Sxp(I+(fyV@>D8EkGM)^OK-=O>^Wjp2ll;5KKHf0Cp z1C$R^euuJ?@*&EHDZfkEMfnKjqm=DQQI1mnneq+Fzfg`*{+04g z%D++Wr2ISOTa<58j#Iuv`7Y&qloOQypnRY51IkItU6dbEeng31MMA*(Piau%7f0;l z>LqlgPoYeuTtJydxfkW$l>1PoQ|?Q-ALag(PRdVF9zeN}(nWb7$`dG0q_imU^HFwrCsUq6 z>8D&oxtMYZWq|Tj%B7U2Q3ffOQJzkD24#ryGn8jio<$j^Tu!-yawTPiauwy-l;==p zQm&>vm-0NyEXwn3>8Vp*P@(6Reel<*ejnbJ-IODgYnU#rT=Y?0{mTc=x{^QdZQEZk zCqMtKl3(;XvG1eIVLsWEYbkr}h0Cc8QeMd8JINXLg67l`<$jmtUS!)%x1)>Y=Gzmx zQ){H`rR=dMc&FyHCwQk8q->(>qx9Glzf-%I?L7Dmv@>nG{GX`u!EJVV9@R)Y2qRyn z&!W79?fF+_F3aD<@-HRl*z94cn&xR#{+{L@o1NOIy|6enSD!LfdFJ@s4l`{t_sbNg+&J)xo>P$tT~ zjOEVlx4E}TJ3{tE>(ojpTPU0DiPx!(QM&92*r}<1XHAvgN;yfHC@;?P66Yh~$Hjl0 z?Z+OLKV(m!PR+C@P^T84O!zUEoXCHxosSbgCyr`p)8o+p6}HEoaGhEw<(~Z5cG~=Y z?BQ|zn`?GE6XpJa@wkKlrd&&;VM0vYd z-dsQa@XgtN)S9H-al2xeA1DVX6Mh^gC-V3GZT|jP!~CKc`06}2Q|`%+L)6dhk2`rh zT~G3s1N419Ccb}oYyASr@345@qAyTo$U;}wA;lGu5!b!cbKx0 zGU112a-!V&J@}!H`Ss(sdzBw@swq|RsQ29Ur14tO&-Ft)j}K5L%6)m3AN+MHR~39B zFuzHDJ18rjQ0L1}?X>XUA?^0uE%sTIwUjmY$ngQn3Cd`T9IvC?L75n@_wAJak*M!G ztgpu|&ofm4VW&31^y&9XzKMDCUzLgSU$)DiE@zbO?ER7CKT7F-QaEvbL*zugFaG=d z{XG9p@mug|NgsGtfBjqH=dfL#2X&Vq?9}p^K2G_s_KSplqWm*d`S4GIx7qArsrW6| zyQJN<-;#QFP!3TJ;`a|#daFZZ4rM3hFl7k8kErYt?W`G>aufTmMETY`>bM6{LJP~U ze_zsfQSP8j^jGtP+4;BG`Ru;FIq#PCw0$J@-S*Fd?RlTzTVSFhyYCaiJYGhbXxBR| zSNiL}KOc3mo}KvZI#vDCX7{_`#O(9l#QLqkKDs@6L!Dlas8K4-%}_rr@Ygao~ac@ zuA@D50qMS)RuNR{&*wNp;-BWihE_7Pg$RL02dJFn5H^Xg=U=APH@+vg#Q zZ^%p7)A@gr{Ul-M*+YIj{ITL$p8uawzDPMl`5NULl_S7Q=XlyH36TEJ3B4I^H+)xZlw6I<>wA5&z(}o~mp5X2d@tUTJ80 z2V$E;(_cZXG&TL$6ivSh@y1k5uUnw$U5J4+O@92K_- z>8J0f>9zZ7`U{8$IW_%Fr;NeTg(Ba}s_zI%= zX>EZQF^sql@p{B>AZ|l^4RHc7^;m7e0f-*NlMzFRYY;C(EJ3V6Y(Q*5>_B`R@p;5o z5dVny7NT|>(j$ft*CSqscnjiI#0L={N8FD18seLXhEGd#`?L=Oh-(n9N9;u$K}ZglL|meRu+56fut2jQBX>2x9ul+J`~Jb%+g!4q zmS`XP5HCZFBQ_#-Aij$DA>y$C*5q+n@9J_CM2ybLjp*o4@LIE45`@Tv$S1n5br=7L_8?0?aD>mia3mT#Byy{ zKH@hJM-Y!)q3tR`d=POA@z|BxuHPdDBHFIahz}!GvCGyhUXy)Vw5oJ-LA*GaU0haN7_X>WR4HE`G$B&tRTL+yr{V5lcZB+ZHgC1ODl`ZO3RD0<7nSMOSeR| zrM&ps6`R(tFRr?zG+vS=WmZ(iODoE&|5@5)NGdgCgEp4c1PUvviWm8ncS<*Gs@PP$ zYVCPzvi-4B3#zILwyavWCOf_)7F%@M=7Oryg7Wz4bJt`CV#_vGlvh-%s;*MDOS9uu z#RVJf&x=mWDFGB^RTLJK%{aa^QoOmeuy_@MoTx-kiN8wuf?-EmzHo6jv9nTqN~fl)VvIUTpWL z)umY*7x@!8P8YShRMp#dOC-LfGSSy&H(@&K$fEgM@*fqs28}|0FDa-hE{avgt5&P( zEW?P{SQ;m^x9bAyw7q^DApJ-O6c)`eTd!Ew`-6J~j5q&ci$| z+E^SfC|eZ3fY^v(x2n9bqNrGnaGd>hc4uD|J#UQ~--&vb*34pdJH&#|u z6;y4JvnZYW6D2=YRiLySgES_oW{rMTrmclG(y_$JKStIv&|w|PzW%2+UhDI-S>){V#jiw6e7dG=sXNzab0 zj@njdbJn26h4T#;+Z~C9Em93DV?|U}|5?kV9Ww?^_OjUK-AAlcp-ARX`y6mSopTPZ zE3T?YOyU3T7+xHkIomJ(ug><}5)euv~1|v=Lj*8Oav=*A*xQ_8>8A;<#UgZ)N9`9iQJ|MhH4+0-SuFP378TdVs<%{QCn>Yns?yp0NE#JTZJbpHIlgpL`5rWL8o9KvqzYG8 zix#7#f}$<4LX~VudBwG}GF>V)4qR1HT0Wm*PNvIbrNpLdRq2ebKjZMs;Z<3twq~DX z;=ry6?wQqNb7DFESdO~vh@O|d6#F!6Xjo)WO@4(1)ib-|)Ug>gUS>~{>N;;mpPfE9 z({gH!N}NN_1=Xb-SC}mvGpQaInnRuW5p+&RWwY zTO8eOVM=J0&8$MKu%jqJlhcMx1yxuJ*Hu)-6DfJ9@S07fRa4o?K@RbRB~N4m?X#+2LrqPrvbd_cq8v+b zd`oPzA4gZHArz}vcNMOUV+GaKr5noU9#z+871gRtToqK57b1UkFQ)KnJ1q`hRDNwK z+O@X08f!XO<%7i}Ruxxy#R^NVu8v=gsiO*J_5tNZOmd}m<8ZG8HGolUeL-oN(oY|; z+b4%`!CO^{kMXk2Q;F#y-0{H3U8nZ+u18#$R<~MvLJx zrMMiuSKZyHz$ zo2RqGX)KIwtlp4~Zl=YeTUJz5l*QC_o(;GQXW!w8on2M2@dEpd?|wYnzWiKXUZgI+ z6YA=DlE^VJpKZ5?EE4rtc|InZilW$>;%im9l?9d6o63q~=M=}$xzH=%?pa6I6sS%$ z>rfOkJ7%GoN9W|0Jv%r1re#fRdG$(lPqC~RcSn#Rk0vS>Da8;#HCAI8TU(5i7u{kX zV%GL&RqA|x;TNzcWpswb#c+E=~+Vzk^Sz+D$Xz7 zSW$(pC^i=q+Vi%3QdLo?#vXEmX8N2c(@P4f%4gT?lB&{p@kQmhsZd>jjf-6dcI9gP z;?`ktj(rzq>S%Vs=9!0T3S-OT=p*Z}OF{M((s~@_9mko6R;&9;@mW^(z28|!&MQ`) zoX$rYKI_m0n~ST~mtmEkX}4CcmgYOAnvCV#ZUSnS9j~awNvkSQ`?aZq_HJcXYWs%B z%pxu>DBCovk-X%?xy-35C@fxwiHrGBqq<$eCN=-yyd_SwI$JZk_SvQA?e;mC9)r`z zrcd74WfcW+blFJ7CY~iZqDJrBV~H+&b`=JT>ccBowW$LuamP^AYU)51?x?TXv~isb zBV>AAaXBiQ7?$U(&cYmzRc>WTv9xmP$g1+{O=<~>Zkal?R-QAcV{+1C%ZrN6uPDO6 z=1AgU5i{)MfSB5c)xgHoioa4m8ndV4YDC&P7^B`Yb7YlP!)x|smeSeC5o~`>xGJln zd;`3zgl#8yQMpCT0_RZ1kmV0JF zwW`@}#}nt4?O0u%y=h%x8Mak3TD-bCr)rbkb!a6IyE3sf&aj&9QnXrIJk`mjj#SQU z&Dzq!616nq4xHLQ%*YGV!yG^I$Zqw=@i|^gY$sSIQq0I$o-pD1a%O4M2W7*c?4woK zdsSiYn&`9H#kh7ZFPt$=O&ytW=BAFR89?PTbznwCr;l;xGj$+G-8`3MQ}bhX8J^f- ztX#FJT0PJN*ylzLUsPVnj+nirT8%2CmL}@FOXql4s*hlLNrq2D;(QND{ z?fjJ4InvLU%Wf8h$ib<>Z?oNr;@k|{N7Nsc$3^9(>VXnw!;5mx4$j#7B@}83n%+aq zJUV^mcRxB$Myu8AUT)8o_C9lF>7wKOGE5}3#n8BQg_%)%i8``Fn9W!1RcQFn=O^??IxD%7LsnHufvvqVxaEvu8yJ^n{(zqakOtq2gCr)*n`1RK8T%rChezF zb06I+2X*0#U9~*WTkKbp@Z7IItU@20*^}(t6OZ5g8{@b`C(ps= zRAc%9&C+UlDkKKyO+CV4r&L=Q-ZY!5JGCf|31d%6+>qI$a%uTG^%amkBn_70ifaCH z=XKtq;D#B`X=l0pykL1jdBr;1jaam}atkgq%jJILGCV?+{kqh_zG=0n3b$tXxJ9|u zKDfB3Vq*dB;uOn;K;r1k2SL-ec;FnXSRb?VnK~N4odf%FWlkRR-eM{$i`pkbN!*57 zggY}E)IH@gwY7|0R(`F0?-61i<(kA(w)t*&EyB%N<)$L+%rWPxCr!9>r`Cni^7Xh) zq8j!IZkVlEg!{XNH8ln6N;lj0daFxuhXosv*i>F~y5w|ETeO4^e-CL7OE?O9iSePuNs`{NGy9&-+C#(kLSN9Uh7KY5_Mx>S7^L%z4795FrZPS<_O z=Bxgl0&qWf_c9VUd8dZjjAE|Z{9h}E=X9P@Y(L`u=*)Au=ro+y-A|-_AF~pdcd^Z- zRq;&)*pQ*8%ucFq`6s?mG^ZQPv0Q|nSt<8w<`}BJJAG?4aBWFJJTW_W8Qr{g^o5nhp&Mf0qqa^ z^a}SLj<^2AP&qw-fAGurFD`FiG3H;*?lLsPUWm0_y1goBUo}i~vJYQFH%`@Va7>S> z=>@LY;V?V%P|J>244j9FD`1;g!=7*?7OWpTOtbBORb%>IMs8++OVMMFQ?+Lt(@WQ| zW9CBjd!*{W3;%WxYc=ZlSJtXKOq~CIA5u>Z*Hg6?^OLMz1LllG+E&MT+AF514Lx!) zdS)N~YR0UUZvluvfqr$ntsv`R#fjWU0`;AU|OBt3QE2iPz_MM*_}z-IM7I z_TuC$^+BpN8QyrDYXisbLq zy`i8t6!5yTA^}+XgZ^kqWgz19xPxAw&l~k*2IBd_fYV)Mc_V>YrMYt5R-?(x^0{){UYFPD47qZ`5$}ZFuDdd_>UGPN>G617uC$COS`U-_NWRP4rbm%@ zPQz*gXsp+31tY;foESQ9R{@g%7qd`@H@8dT1y*Zp>O&ANLJ*=5X^ zUB=ApGE{b!&#!V*K2dQ_(_E_P-Hw%5Udxlwpf~74dMIE;qMi;t(qLHmcBkpl!^&lo zdI+5)qfB*@nSG?z=d)@8+YN7^Uw6CR85wTxcEdekgx#KSmywa%X`o+(&}a$r$?DSU z3@bBY`2((iD&CplvLco%D^%kTXHDwp896gMhU#F$dcc<>-NMzQcj`Oz%IycbEx)y! z=R(t`;S+n!O4J4pnyO9s6W3(?|8z}uq~u}%gvOO?vSzwQjhR8XCX24=GTQY~JraRu zT2ixe63v?Fdgk`k=!m5U8VXa#$8%kA~K+}?o8iJ_0;{W-Y-m)!v^t4(ikINk1m+v>yQ z;J4gmZl4>2!opvl2#bT*5$M40k z!_3vXuN8=TOT10GH^=J@Tfy9Z%tT?o+tZ?3L6m_|qW(;`$mNQ}txPpMYih0ON!6~q zYrqJ=&vxxyP`ie8mwO0JbbB&`I4xek)#N~Hd??TDj(FVxt0*+Npx4o5^rU#|FtYOe zp^TtMjW4&`k7{Acb$LA2M4Dy#z1s}8+sbkysn-g*t#Bx;^1yrupZK$^tWek!K^wj9 zX2-Bu;&$a)F4Pl_j@mN|W`S1S=MBIW;_9FpYaSPh@Vf$@h`UV>)a~VRyL~9Z-=zw( ziwcDBNfj2DU6xN3T$-Ymc0gEQNqTmBrEKL@=70di`#@M2ufm@5J@ zS@;9iZu3PqW>Z(FUC(Jvbp`TTOkay>VP*}5T1>aM(M0D8=LdFMO+tYv90(73tccI! z&vbiy;-fIuoy;a3D~_rFXIo5-_T1$6=b@hVWN2A#cs`JyAC+!v75Q5oIOq;|a)PRN z1U+8ynB{Iq=dOow4^Ez%7-dTA(J>DO%KY$e*v@^bPiFb2{ow9LH7YG{e#m7TgP!oL60Jm;mM723$#iA5IIIEPXCFZ)_Mwe#Prx1VN8JH$#O-pqK7Q=i z8y)(H-sI>>!|30JmRX)g-DfwQqts(>6zqOHs%Q19R(0tUx^fRzWvqyaQy8)$o;+23 zk5%Kv*b87g67~ln`*l|!(5*)U_JTEz1xqb+SmpdKyTht=Zq8B|30NgrnWzrt4Yw%ruxP&^xi4KsU!w zz`8JZZnB&0#VAHPU%T3V)XbVmF)ZqhfDbz(d-LIHcLav@8fQ0lQFVH-!gTLtWbDugmzH zx}Z#N)Ss*VpMO1_>Xb8=4NT7ydZ1PJqoaB(k85hCz}%JCU+s3%R7VoSfUiXlxVv;; za3sAZSfd8+o{HH+2bj)FZR%8^iJUR{gfW%(rSzGY#Rtq0N4&${GR|4d?E%wngrfe; zK1_R`tOaxTvH$vREQnr+eRoOLP@hvl zOdV1xoO55mstF|=ome#TLo*f&^at#EJ)Quz>OpKGT<&f)7d7m5DTFN=HWZwUh8!b$ zpV4fLr(+7LH`Vmhd{D#w_6*?5cll<`0nU7vn!9BOrD`?%)WJu2ve0B$9svHg>i3>^ zM!<$&CU zwks6!=3=pcGs9Snw(~j_3t<>bp}On|1#n^F^F-_Q>9MUGx=qJ*t-7|3hNXj{cVSMn zhOiY6tG+dL@vSa}ag`kk*t--fab=wO&!(ywV86YB-Gg{pAh?5j+7 zI4dmwXF<&RPu;*km&Y9p`?iSJg=^PnCXQls21Do#)1RS=ssiMMPMPqq*aLP?5R?@5 z-I6IYm6a+uq>5C~I1@x8ie{w<2by(%!qh&3n?yEe)#-3jcxJG(MhCR(I9q1UuFPy@ zgPIvp^@!A`t z$n5=5WX|PDGcHdsM`5#TS)F<)zX5~4U+;0K9k#t%U>d+u9QHP)hAlVNl2J#{C(B6N zL4i!GELjrSlF#lz=vz|sk@uACcVr?wR!hvwqoE_TFMTcC?jLUF|fwU!d33} z!=mWQb>889>z(!P;nd;OUbENiJhX1F?!9`=rj+ezJ$p~2cBLszS6Z(*d`RcMgL`fJ z)NpFgg0_PvKkadaLQS~O6$sY4$B(wQqk7{G+yX)ON5IwJj1Y=+=wtSOt=JHRLb={d zD=Q=D#ggK7d9XhZ#l2S4X3Xxm(J|+cI>@7C(}rr3t|nr2Wds*1PdDk2Om$-X?or>2 z^6iiAZF{SF`Q5$gV0>uN>vebkq1L!z=b(-ystqk{++B`A)4x;qyWJxOZU?(}=mVxb zj?~G+g+%k=R*S=>h9MIVV6aLi$YEG_kDCLgd%)bbz_KQETyzK3I^U!^xf;;diWcer<$DB9iYaQCGsQf)??2V^t;Cl z_o%}?;#N(=0hI*pQTgMO@_-WC7QoKqai#A}8#Z^OSfl#}>hvtD%!;&_m>06L(8VoJ ze$dLYTse8$9O&F3FD|%qGcxk(v8C$K(TDr=F!mtQsbyc`R<{O1xNOSuS>9PY3!ITm zkJl$xs6jtA+ z`WQ0W^_@nmY1IzsJ;pHZF!r6`bvv!b!-pMXDfN3}5%T75Gdj$nv_NDs10#Lhtjtt> z!&6pgjOay8#!d`k+;UgXA>8hgfPYNisSi4GM)vUp{q1_AfoB%DF@V#RkK69}fQ@~Q zJDTJ7hZ+s6_-^d1;9gH&jk<3Y3cHE|R%VOt4~-f*ViHg$eoquPhH3}(F4be&j2bs? zEd+-h-FjJWz~1&Y7@kSp8NkyJ>7_VVp-9Mu>#I;GtITqt(`SXEUaUeRh8jX1`_F{w zcV=MO?J{>_Uv4!!tOnEVMz`2@GP(iQz_OwN9nTI*Cij_08`B5$ffV$NdIzpDhf)Vp zCQ=$5?WtXQi{7?Z_kv+eQ{85}-VV?7>A_CD1lP5mNRK{jVu6AUdQ3^E%a}A89ld(5 zj%;v)5Nmu0_W^O~kejP+q-VHG+EGw58airZH0#pl8gx=jAgKOy<->uXt)xyg*wpsJ zOC^=qX2X&2W7K6^&G6b>pG|oTH|sDdgr?nT%2MZ$KYH zw=U|^-Eo|jCVd=@Q@gyTv`}6sH_}@#OT>u<6R9mojH%aK6Yt$G z$Lc<~->miJj_Wxsx)sgCP1l-W)CX_2nH^X*id=zUNw*#f2VK3mS%TZ;)))q~{5 zfwDmE=zds^fRSh;EGDUe6#b=t&9!bg#F z@WfPwi^zlIo5>lci2gZpl$>^g zKNV$%3&@$|Tgh?q|Jtl+ZR9J^E|nfX`(%e9ayvPMb}9W3xz=Vp?p`YPs+~%|ZJF?0 zWZ&t+?~(J!r=s4`r`*!kenUAy<=I$&Zk`$$uj6ATL8dQ0d3XFOvf+B>!V^UX{L|Tu+X!6#d>f zZ%SWAzLDHQHsN=r-%h@PypxQd3$yjnh~#$y{HpAW$d8lT$@{}kO5a0XN**EKOHNxQ z`8`X{AZuYsUrAm}ZX;KcyUE`n_mW>Fd(PJE@DVwO?1djy{aVNcWc>7)9h%4)=Lr9S zjGrR2!@S8Nm3=e$d*nW{3w}|0bG6u)kc-H_Bj=ne`m$AG zUqk){xtqKW?NIrRk>4ac&y(~^QBS4MB41CgBma!tPTmLgRQ5gOhskD^?k{kdL}Z(hrk2kn>@uLKnG?JWAe1KKWuvukux)mYfbd6?(`4@=kId`HV{> z{RsIB97#{ ziG1*7l74`E6?vTeU2;Cssqh_6DY>8gV{$Fhsqh}Tg?w_Jq#q;~lha_Q z!u{kd@_tu{y@ul|Tu9C*ZzXq-54=+Bx0CN8o3K;id9t5;WK8VK$fe{l@)P7Nq*Ec3 zFZT81IJuA9NggM^L3Se_6;3RW^d;mN*`IH#wJ(wL$osDo`&RNAvIRR8ZXxd^e@IS; zRH39$(g(@6k&DO!2-%NxDp&bs3w~)_VFZP4vtz-@P zs_+N0MGkBb`$}>pxsm(?xs&|q60!H7Tov$>5cd2VB;&V6Y_3H)DtwOIL7pTJleb(Y z_T%K})xwP^SB00!UF56EM4y9lRro2nmb`MK=sUUL#ybt|qsUcagV~uc;FIo#f1FVHet^!e(+l`B&s-@)>cl z?HEpgk%!1P)rh_u zep2B{@+f)r7SZp3UsR|ghv63$_PI{<4XC#Yi^*=(Lxp;B7WrOs4S5W_pRQ@`6)@=hn`lfO)EB@d9dkr#eJ z(vOp?$Z3s|-^=7o^658-eIxnnd9g9HgXl&cdw*xBZtW2 zVAY@V$tL=N%1_%W_Lq~Rg?^#zyU9J|(AOlt!*Sk}K1#lgTuUAzH<7(KFUmgs zK1sic945a@ZYN)X^Pubp$loWAlPAgUuSh+Il8A>TqCA$OBA+9druj{JuLZEw~Kv<{6}&b`NI1}KSsWf?EJ2zcYaIsS>$YTKKUVXBY6SNlU+Y@3Av|B z@_Uu+d_?&44zUlCzeR2$J8+&=etqOhvgc7r{|Y%sJ_P4k*=LdS$hqVf$aQ2N&ZDyL zB!83KM?MheN$H*6lk&bn&Pp)+uJjG$mGGnDHu7WSZt~Iai_-5TKSOqPOa4XhlhTLD zljI`uSKtSwA0Qj3r{WRvy=3S2CBH*aj?xFn-zMiL80nP0iM$X}vGXxWeSL`kF#pGP_ zgXDVhhvYQWONGupNgpI{`LS>V`S0Xz@^ep#ekXbPPlPkj4i#P|`^gXWi@u+H=F`H% zz)@LColS`a2)Mcp_SZ1uG%j8LGnxF zQS$9S6TPd?POe?>f^dZF`nhlzep2BLatrz40nzu7um6Rx8-7$F^rCPkdEb|WJIKEz z_mP8xq8}vROWr}Qcv}>B;B+O}K@8`cC0q@;-kTo*)mCYtZji=zdG|-Q>o%g$Kw5|70N6C3)7si@{$iEy@qkDLOt0>B9D?kAWxFd#J>us z)=xj?Ar&4WHOOLsc@hn_Ic#bk#~?s$r|Q06;?RJK9l?)IhTB> zDf&io4Y`~AB6*m+JVoq_FfXd`0J(;Ic&g~z$ybqk$Uh`+C-1vJ>^01zD%?&kA%8$_ zCSRN;_C4fovWa%x2qFCaZNrg>TghJ}kCFdC&c!;T!g&XXeLeZBWHMxy^&OxG2-y!zDBnQc*2aCRp`~kU@ ze2rW5gJk^>VK3G}70Stt&`U0Qmv0=*P&He_A+gRPx*R zSYZ!&1vx^#m)!Jcv457_K~6o6>Hi}7736&KHRKxdedHGMtK=bay^r<#tK|18*-K76 zUi7u(E^^14VxM|~==;f^A!~mVeViO1?;=-{pFL61x07$TgontVJ4rZwr{q_9vT&Gu z?kU1?@@anI4zhQV@G$v9vJ?9x6<%8``Uv^1CBk*&3j@O4 zl>9Wgo_ykIqVFP~16KVr|7}TMOnv^i@YCc*@&Q3fKS*vMyRlDG;Z1Uce9AJhkCT5u z?jSEdUG%-=`^Y=VE6)&pF7|yYyg;repBoZ=2f3a+NFE@MkoW$K*lX`eesQvkJVp+a z%gz-0dU89ti#$Q@Coeur?48(Gs!&PJB0oUxCy$ZG$>)Y8eE|DR6>cQgkq=)k`r7wJ ze>u5>yn{SUE?gn@7;r^84f>^5<8HeK+|n@+f)B z*`jx0pQ*waIYKTwNA$VmUy?h>C7Gh%P97%*u@6-tf3@hd$UDh3 zg(u0UT`e3uSn^*{CLARnxly=?{0_N|?5GfZKY5=@;hp4tuMu{++5RfwI`W^$&E(N) z(f5&GiVKgBpWh_xJVf#fTq_(V?_I<6ZPFFKUGc z$pfDgPCHD}fA4zX5c%hyXL|B4>V&tG`@bN(ll;{igslw8uj-4!on*WLV(&*s$tQnF z*gRb9_qtKoMShJOCb!)r`ZDqt8iZTOXWuN`PyPXUjQlII`v@uTtXsrBLcW9CNZ#*O z(GQTXBTtflOty}c{4Qt|`+V{MIZj^jWzlz#|4L5xNc!t<6Mc}ZeMLBr940rA?;xvv zx(Y9nyU8n>B>gsWA927W*M`8TpD9(YKIa zAa5gQ-7ETG^7qNsv65e8tLUTT7s$Qj;8#UIKz@ciNj|ex^q%7+zis46@+n^vz27JL zyU4lZWA77vC;1lg5c$_+^LR;beO>GW{RsIv@&x(x{}FxH36g&W zIqyW_c5=uPPW=Y!OD-X|ohJH5au2zaJVM@1b_K=$`fo~pVKQFpv*(vevVOmCANh0S zG4iF~61{Jk1XTQtz^@WMv%W8!Nj~Z^;STcO$bIDJejxf`@~u6>X=h1(!d43wMxRKNapLAG%$5oP6BRgtJyk`omrjt|PmD zF5FH22YHmd@E1%Uk@O2+6s{qgFA2Aje?=Z5KRhV<+*OkPmY0Q_$?IPc?joP{OW{89 z0Yk!L%UrTN!caVQa z9wKl2jo7DWO8POfhn)AC=yS*^zZLEwH;w;^cibJ zezfjU& z@`mVh$v-F8lh^!3^n>Il$Z2aOeP~Sd0rCUnBJweR6@3l4fxL~p@J-Q=k?$qvWlR1a zk!#5}|4r;W$h*kf$xrPReOr#C|McI52gx^)C&}4wi9YipvHu=9PS)OLe&qAX1LOzE zbhBlGV)Gx z13BX|(YKSc$b;mrD@4DOY+fmh-yyO06Mpj7$Tj4J{}6pIc?)@z{5aVill)e^FZNmF zACdFP$A2LDX7UTEf zk#MRm>GQ~kgZG1~iM)dPQSx@`-5VtRpKU$f_a>iVNcu|h9pqMWA33K)((fQQkavMq zd%DS69g6Y(;?-jRbDPm#@*lvezBS~%Oi7(_C%;0@Bfmp#BOjeA^&KI9mh7yQ{GKL<$onr4 z`zCS`xtIKP@+f(NJV`!3P11+1k^FBamyn+(HjglAaE$ph2 z{By`*^8b*V$fxcj_PyjS?6NG&LMw#f3Yti7n9q`caaCkPm?FfpK^*notvb*MPTK>Hu5#(?d0cZKS6eX zO6mHZLebFJ9t93bh#{0_N?y!s$XKSI8ZoOYe0|06j-UUIP5=acKm zE#!UOqK|%7(r+c#l6R4t$(e_UeGj>dyo3B#@;LdJL&ZM5R`TCK_LH}hi^ze)#J+}H zNNynCM(!s6ki3m-WJvl+@@L6KpOf-mB{z{zI9%+z$v+^EkXIie`r!4F{wZ=3dHIo| z?y=1U zkjKaqWb;d6fBtb|UrBByH;~^Ucahil#C{w3JLGY4%JHIi-zfRV$T{R6lIzF|P7wPJ z@+IUxayNO5JVAEdB>5MfDCxuG-;?vnCt9L!CbyF($&QmmkDutV&u=z4i+mq>8~Hu* zPVyBeOM36klKydW7WtS{L|;RGki3)Z@{2y>7D;~z*-vgIN63F9*O2oUN%~=OA35z- z$?sE(MQ@SM0;~05lAKQtHi~^S?aRn-Q{O`NE|L8D$=@VreOb~k35b4>{3KYF*Lj=h z4>?uzqvS`yN}ur+(eJfXIFnoi-Vdrq@^8q!Bq^&*(~pFDgUSBFgf)cmPbAwtm-hCBH~k_x)8k z-~uVHfqV_w^G(rzlbl6f85R3_@+7&7{P2aMA0o%r3gegW?D9S!tLFqN+>YH8m_IqhXj#hEuC1htbI}OopYSVP)ko zj7G!MVHk$dWHF2;!^zT>(W#|j{Y;Lo{C3{2?!V{b{eFMGH$Q29*;L}W*l&yW=Wqmm za;*9Z_+wm+tFR$Y`=R61ulv5h_dnj{I1*pLMjZ5x`ssKruEGD}fUUYW*sOjeK2>MW zkDn38jF)q_slOUmU>A1b5$~!Wv|athI2zaBc>Kf!^{sdTuEpgzFkknYB&uKcmww#; z@%R+SVHYmPy(X$3vqSwgI0d(wq>n!XkHqGk>aWA8*lV)-nb?dIcd362XW_m{>gVG% zb@u%Dmf{Lrg_}=NUiY_y-T(NaaNYNEyzeBOiI?JH{55vtpK#!A?eF1O{P89V8b11-@;qFFBMNn|bcS*h9-AVk;Bhl$D{ebWw&Sm`3rEdX z?z>O-Z{aBXd#dszJnVh$;RV=tzxIc)&)0IZIqH`llt*A!vAhN6AC}MIa@^(v?PE)n zC*ulSfWwX`{{bi9w=LS|;B~kN*Wg;*dan9@M|mEu#<@87m~uOg$9v~#pNwaJC}-eH zxCpzc=>uc^~7mmV<4k}N< zuNTXe_`z>vpUb+JhePqmL(0wg6wbykA68zDpDvNTe$l;VN91t45XWJ^qskNUTx`WP zI2R`!)8|8l_yn%PJxi6lu?2@Z^?nZFSlsM5pO4{Tb@qIIm50B=F;}$jaYDHTU&n^4 z%0s?Y?uV^71nz~HD<{=2#?x^P?)shbz-qmpB{&@4#<6(BDfLtF zMx2A|{@%y?KOTj6J}$?1a4ml4G|#`L=O4jF{Nx$sR{S-#;qbG{i}4Cvjc?+R>$-o~ zuD%(!Iw#xk0$hym;#xfId-Z*8=-zG|kK3MCZoxBf4c?EvYqWo~Lj5p28pq*-I2|wk zL47-}e?d0ORmNKc*#%7lkkH- z%h`A=uExJ%-`l$P)@eEvwf5g@JA*cG`cXhA+71@l};Vj(sSLKEHEOz1}zbW_qTle->$XhPpTRuKE%90=D7T|5k3t-(&Aa+7EXt566W#3D^Hec@BOG z+wlNw@YcPfI2H%~t9>C}f`cDazYNFYcK3K5F2W`F7IxxJhWhva{pH419M)L(PvAK0 z^?>>o{4%!T4{%Tu?N8zu{8&BhlkxjF2j9W|KH9%nU;Q#X8OJ=Nd94$xxB9`4$~iazCq1Y<2itLB8}<7&R$h+3!nOGECdv)%)IWeDaEOm`Gfu&o_$zG3 zJswg&>@nS&i_Q2B&c^RFRlgG7!wf#z%H!~OoP@VxD{jz6 z{X*Oydk5>@b{v8q3{XEC$KV3I8CT-}VXtR&@8!1IN8!CV9mlm(9`UUDKjJjp?=j_m z&nf>2$KdAzm8avGxCkG|$syX0d|dqkydHaXP~PMT<-vFy&cvIs4L{gkeLEhCU3fbV zdS3TG2vXmOuj4d4=1Juad=UG-!25ejc?KSf3-Dp=z#X1e-#b+M-PnSg1nbxHd>m0{ z&#%{&_zfJR-UI_^A%+XX1sp z6kowHU9|7_yq=ecH{vYp{ep5geglVe)!vCq!{xA0^=t8b9N10yRUD4{cT_(XZ^3D} zStorwx%hqT!dvU?`SA>WN%#N6Nw{-o<@q=nSK>|Bg-_$q?z-0?jQ58};bgoW=i*i` zs$Yl);cEOL_KDE_5*&-&*o-5)=w2pH!+H1|j_sj)H*qQs>8iaAkHsN9)n9;3_z=#- z|KgaJ)qg!)_w4u^93QFtscy={ddVr+fe+!p-pXHmN&VnH@@#CxN3c(n^3L7W_wFkn z$5HsD2<0hw6?WpQ*sq`V4SJ{_jRSEi9*gtwW?X}7aAJSmi|VO+nK&C4VZ+PHtMI%! zd;a-0MQeW%SK&dC$~}Ls#PiyNqhFER_EMgPGjSoljh*=I-s*=A)P6TMVgEkLEjS4W z$EaV76LGgF^^*rFpO16!57_Xk@@{?A_rvRPIQ|vK;x7H@57xd68*#J#%5(9%*e_Q7 z<2VvGk5)ecFTf@Ei2=$DL$rS%`{Co*irc@Uej%QVEAb!LXQ=K?8>oI7{uq~FTb7aY{l_IwJ*nwhskbi!C@n`{~DX{SflzGcndDWH?Ys! z+IJeRegqzmjra>}!48~*cfQ8^!@u5l-QPDbQuiNtU3n@Vc;5y?2_92t&p+=Kcs6;U zN&98&3rESvuoK_L1*4VsiqrEPco7a5qx>w6$F1K`KLZcPxp*lq$JcNzj($`7fCSxN zfTQsxoP^uFrM?vp#zlB3HjLH1GVG6I;zjeo#?@95q`Chdc9FC2lV<5=8%l=@L-?cc{qxZ!B!x%d-YgrmnO z_a3kPc^rb@OHf{pFJaH0Bl5h)j8z`=u3U|i@tASSZTLF2cnWsoa$NhN z`d+E(2d2rxa6I0Q)A3zgfCs#oC-;1O1T^x^} z|4{u3oP-S@Yrh8v;)ZFwADoP<@FVk;2Q1RQACAVS>g@UaBpLrfZo`i+Q0~Byb@u%J z+bcu&W{`*Da`F`1aH0A(`~Z@Eu%$2Y#fz zd9n6ua3PNPSa~(RizBS+FImJMznLND;y-aY&izEW6Hok9u2`abZ5PWmxEcp7Rldur zJQ*)vBIje*Qn?ztm&snsv@gn(Bk|$oasn<{AzSgXm2xqjoF&_F{3_XnqgKnFKPTsT z`Fti@unXtm@3NJbJgnd@1-)^1qKztI%;MDcXQ*oya za>`2W@8EL0Emye zMR+|9TCID7wkS`+Zd{0W<|%h#^H$mSGd`Yeas;#VSKwltv!DK2^*ekehvWUY z6mKn3?!sRlkYhjBKK*Ms4NpBN7vjibxf=KXMmFYXA9hH#;O2+rTx`dcctweF(>m=( z9+9(g@KL!EU&mhS)!%VUc_nTbnLxR{fjsjk36M*CO(CWaqDvBH8>9keW80DPb&|{ zU*bgk+8O0`{0BCCsr{6*%7gH39E+RSm7DNjoQe-)EAD$v{UUq{8*F;s!`~|p!IN<$ z-i0%8oAc_I;ZfLolkTm=VfZ9A;m0bpPsU?$Auh(%*#8IhlQ!%AG;GD+;vD?k1@-NC zCU)ZkICP8db^KBNEW8Mp;O}u2e#N1FaGv((u@S%blk#+2|7STHug6|nwf`N5;)ILp z$KyvT zU*!~h7w6!4zbP-lPglt%Id=<0pUTJ|2s6@kVUNzhT1;-FxP$_EFe^Q?MPo z@pFHuAFxyVDL4Ti!O8gXYV~b+0xripunYTKQ{S>n_f0qx7vo}l2itMjb?vKgG7c!v zJtwx{mu{$^hv(vQ{0FYZgKO0H->rMQaR~1Dr*acE{7){$He8E)-Q?aL-TM!tbP`riS76_Hk{V}ktXVg;33$J^KtMQ^_%&qUyKv5@vQQL z*os3QQr~D-J`-o-)3_8r(o}sHo``+UX}=mr;$zr^J2unaf#+cF@3p^cAL zuEhZlD-S%cy$Q$Q&u|KUp}G2bcq6XBZ5~nXU7>qPI1C@bCfvnO{bW1~XW-qq5c{-H zzYIrVH_pS^Kj`@txB%C0sl6R{$Nm@8e;0@1Hm!Kx&&uDxmG}U5;fvV!qWXROwGYRq zaXfDGsPbey183m%xCmdvmAHLt?Opf^4!xx3zuZQ7G~S3!xJiKW3_J|y<3qRu6}a!y z%7cE@y?rWAYvY{I*6ZI$}XpH)Bh4|y{-oQ^Ny3OpuMeK$Uj&DV7= ztfTT&ycOH9Zztttcn)^so7i|m_Xc-XzY6cgzBS7K#$nhLroI_3!kKsvw&8o&fm2@8 zKI%{1KaZ2}(_NHj;`z7$U%}P5Q&;ta{-=8faS@IQS6+pG!2UPYpU_QtB0h|>aD$hW z+i_QHsMUTv4#9hIBz~Z~_Br@ia1=g|&A8Xg>X+iP*z1n=L6O|U<8UV4gNtzEUg|sXyV&=x?p5J* zJg~R=)i@gm{H^}eeUy8?F2nPhft|P%`~IW6LzMc#_+uQ2uj6Dqw6FS^*p5r_YyFh_ z{Hy!>aU^cnUwHyf!?pMdw%^mfceMJxhWhtEo*g&_Hy)ro1;2xf@E%-?f5)K@=w84p zydOLrr{Tjmq@Mci2CAQo$6y<_Vek6t{}iLX!ApK(kQ{<{;W(W2s`8Qs>e~m)&W3Vf ztZZu}j~^lz zxDvZ?`bgzLKH9f7$K$_i3qn=W!%%Ia#?0ufoZ= zW0LX={2ngG4jkS}_aC34egU3|EAb6ni~CPiKhR(MU$7C6pQhY`8zgfd7h|tSwKq;z z9*SGPCnw@)vTxh}X9QLS`G|cx-EZ-+T!_EHPCRjua=*va56qB#17#;R;}xGMw_@X`auu$>SoVKh`$ISd z&$6<|J(tLKd<=U(q5b5g%EPb=J8;f2<-zUMkI9si@I9Q3cP>|6ip?wJTHJ1>92lf~ zf8a3OV-@#t=xR9~-@@5=|7Xg}aZ0xA^Q7)iS|f+zx7W(?IRA4w15eM9i*V<4a>P@* z=fWxYz=oKl&9h0TjgThW}EEwoccfGV7zv_aw8s)FI(`#JLEik13Pfa zPUZ2>>%JYk@%y`!hrXb^c>#U=0j|ZrVMnO?(Yw|6?J++?2|+nM)=Jzv-3dCkXJ_#0e=+wIrhfs=4rnD+Z|Demx< z`sFVwpN2hOr{j5jgX6o%ZHm-S!gFvYF2Z@Z;Q{(xwO@hr@mXvLS04N|eLNRu;}Tqq zI~`O%u$%U!I3Bkx<{ln{tMOj!_mcL_zEM8{Ps3(>3s>Sd4yo_cUHdE?h#gU_E+9uTQ}d%u?> z@rLtqBKG}3&c+vTKriimF0jWx;Z)q$49LlS4*PrC7-n#GoGxz(*$FLbM zy{O#tbw{4pz)JSG)g?JPO1T4@u-9ef$v6aC@ISZ&FaJe-7mjhtL49@KiA(TjSCsqq zQ*QiKj>CSx$!XY*^YO|mvg#h*SjIt z;s!Ocf3%)g?@u`jkNTgSj62_ybMe18Xn^+RwaQI+{Vn>q`(JVew&Su_v_E}Yxf`Ez z$;N@oi|@$kxZti_gfsq@Yw#Mk92cW~=09>8PW@N5;pls^6Z;t&-2d;Z34^r%4d>wf z4=8uykL$_auc~jXFNfeJUUDqHh?DX02FkPW$cAziez%eAKUnuuz2zkQ@Pl$OZq`_? z!G2BT=veJn<1Bmtd%n)g^9u4&-+zdlf=&1q&c&l2QokH;#)hHVcWbIV41a~A@$=1; zr{W9Pi4%R5`wr8+8#oMyKCIk?Kg3D61ZUtEo2#FL3ve0k{s_-A>Upzp1U`pjaZ5k- zv+-=~z}vC^aNRqHqwwE23HNBBduezIF2(z?8yi}xZ+T7k2jCn$7gyn9*o9lP(mv#M z?T26^{s^aHJNAlGzqP;i{x}gw;Z-;eAH|8-g>!KHqq^tyhVFlXL-8eS#QR#SAM>XA zquR(8JS#vBcuRTnw)F8nY{9RzQ*Of#JtjMFId)@npz`2&-S>T5j>ZddJU)S)IPeMe z6Gv!28ArV>e~m4;ZF}_#uo+k4y*Oy3_J83R+$)IZ;knq)r2ak}hy9;a-!w{j49>vm zxEhyZ!)W!}Kc#&T9)iR1N4N$*__X?=W3=yrEqFGr#wFN4LH*k}0uKw;y=?pguD}7$ zD6ht`*gRJEmSQWez=b&YS?!&81&$x5{jWF!$3CZi4o=5*T#Re+10m|CzN34?v6oq1 zj~)0P_8qT0qJ#F~*zb9{93R78@2Wrc1?2&_L#SMaOK>eVc2piRLHlfMogzQbNxAn_ zxi5~yALByYqqF)II2)U$X-1$Y_%fvIW|8(sS z;c$ExTX19-?XCDzT!K&FYFw|Y`hM@}{ve!MPsR^BW^ z{WxsIC3p?4!R6R5Rr`=0+9%_kI3M4_CHR@1>PNn>{RC{rJFpc$@Ur?vcrp%~qy1r= zfICL2pModiJiG~);j7sAf$sI}rM&|$!UY!P7jO;k+FSjgxyt9^SbQ31;--E0c<>w8 zfmh+QdAjGoJ|D`RqO{M%b8%vt@=Bb8AM4xT{@0_G;x4!r55{5hwKroEo{Mwv+B&zZ z`+Buvd;?eG$bO#OVDMj{d)w-K|LgVB<@)`#_g^HB#_{-Iojv_@Y>w8w0(=~MeX4zj z0m`cu%f|b*7~-vRA@*7#KlqBCABkVViFgst!c{mAcN?gE4gMSlE!F*cG0JV&j7#wW z?7K|;z(MK<;zihqoj4VDdsY2hya9(~>R!Xa$|Lb~oQMx#3vLvvekOhmm*R6cWV!CY zG(`P4Y{jYg2F_Wb{%b?kci@dUWTo;KhAEH7OK~oK$f!IaOa0ez4qk`txWRDltx|s$ zcH*nJXf^NeHT7NCjzd0E9`U;JIBdh&xM7^~5)unPrzGo3U2e3@@$-d^YK1x{#^Hr@#@=fHg;h*PRLQ;G(vqV-i8bC({C&HTc>_0 zj>W+vl_%htxDr=j@AcYuH>n?rH{*ERb(DU6u;WVXwLyE^Xyry6I!3nO`U!FoUWBXg z3uBeLalLV}HCOjHU>i<-N4W#PYL$5;7q;W-xylWBx>q(&4#I^W%CWfBeA$98W5ZVM3l=DkzzY`2arnb@IUO(jNUp?F zKbHNr>E5_Say0IlA*bTOpRmXMK9zm8YybRWITYW;iTJ2hc{N_OL=Mi^K5?lWg(H^9 zM*Lu=Y{B2)JiKDL@(Mh6h3vaS_gk)%gK!5Ni~C_K9$RP6=PNdR4BPQDS)RP^|ESxk z`;m3_eEwm^>&Of6idDMjvrqkhaU>qFTDb);#l`pnuE8xoQ{Q{P?hV1gI3!zn5zfLj z_%aUqO8aNmc=~mJzaj2>-*tcg_5tNHu@PUwnK)^!o>z?R*oEUiS041W?rp*G*o8Ck z)Ew^NE4T``U#HyfpzbZdZ-XHcZzgvfR{k^gE|HtA*S-Rez=k8rKf&er1a{*_8}z@o z=j(huuXc6z{CsiacgRDJDSwYV0dK<@_zEt-5xKhO!qaeIsqUS?3D|d|`Z+l2z72*l z{1&;-aqYh$55*4hfNz!e{zCVn@jC2Nro7RY%ER$woQn72V*D4bz|Y#W_d2P2!*L|e z!0GrfF2HVFg$ta~mz1xK9H{rxxz8@8xliw9u8 za`orpczg^e;ud+@=ini@6ldeu)4F#VC*ZKH+FS7`oQprjg}4}3h*PQiT&)X&4qu>;@60TsG8ez*G3_%ED_&3lwv@h$AaZ|+qd^n>o* z#xZzBp>h-c8)x9ueadt2&$tY~vtPLj*Wl0#dfv*fl-qG=k?h8&u-}jBPdcDH0-whw zJnn1dnRpj2#6MvtZgfz6KZowO#*w%KPQxiUA0NgJ+_YHtynoWYIXDbo$B8)g8})sE zR^N@o@w`LIW3kU+IR%?=Hokz%amy0*D{&0=x~S(_a4_D3Be35Q?UV3&T!C92Rqj=( zd$VycHXKtPiwELV{1vw0=Sq3Mcppx?r2E~DE6>9RaS0x9LU}b_gMBY+-|AcCQFs_O z;*W3&K7h0FO`MO1lTFc>?ZqMtv(@j=g`?{y2`pK4;Z0!=thBH}!Ym zkSf{NuD%Hm#_4!54*gyIV>lT7 z=hZL87F>(3czbj9{U*Ia-`l@nom-ahx7!LbGc^qz4EhpYlKON^_ z7cRw@ud%LTif2)59oAA&Y^{x0@oQs$LsXX-`?O*tx zZ2DI|j&ttGFWyvMVDP&C{?_5(M)D~fiSOV<+`d-(Y`g&HJgEHvT!w4@>o->3>6Z4T zcrkXgP#*V}@`SeXVr<1naT&gbUD)ro_Cf8me;tS871)H&;bI)<(%z2W!tVd}IPx*w zzl);-<&Jl>Psa0c7B0Yf_y$gRT>GTE+UMf6xE4FH=?V3p|6BbGJR0ZYEw~srcB>!Q zUi)wyh7aN-+~6Pe({Kbf1Zh7P2jeq19=G|IJx;>;_!k`fB=6^*`bKQVX1oSx<9ddM z_kTZccuM<0I0S!$WASNRfuDUqd(+d}&%xorayd@Mo$9G?!xn7EXR!Y>+WXa4KN63| zCcF`QKdZjMOZ_rD4ZCo=2Fi`kss9?z#2MI*4`Z(o^{?Xq+^eDPMdMjG5%0k1xEg!D z&fD{9*+_dkeiLVPP`(wH;6HKt^UC{pYhQqu<4XL{gUX$q)$iO`c7)0Eu=k5{Ee^xS zo2Z}Dh1^Fj?kXRBNKOlvM>UnR@jYCOS2j~#jT`&Qf!(yv#PPV(!^$oA6wbl}nk#qW z!#MaQ-E-kM9QBC$sW=V$bXWfb4#my=)Q`feaT0#Kh4Osdrlnks|Hh#ae0;5xN8_&k zvI%d%`S`_0mD}+?9N9zno@%Y!h__=4exi-?T)YX}@rD59fjxDvS6ev-U&NlT%l5oJ zZ>QXWTRq19W#z|lJf0n>ya->ymADrBL~8%ysdY&cQ$9GW^UF+WYj< zejX0Tf8j(txV`!od>DJa?%VTvBuKemZ+Q%k#HX+kw|-LnOgs%2@R; zfRnJ-)5^1PIxfLYgOyj{x3O20?w!E?*z}D0q1c5@c;&Oo3vt`$k8yDfHUsvwNK5=sRNZmh= zQ}J7GD9^;d<2)SsCin0G9B9(LkhhduaT>N^7q;Wq;?;NIQXD)=_qL8uo{YP^E$85a zxEjAZQhDfT?N8%mJk_K;A3rimcHtE`c#QUeqm^6m66~9xyzv<2(ReYg!Vw9|E61w; z>{vNEn<76C< z>rYT_#Y=HMcH>gKDpCEE3A*pbIXG^j@(O$&yKwL%<>86iFTu(98qUIDlevdyV*iQS zAHdQ07B=CKB<(GDId)^ODasQk>E1}3hIilsd=*#W_EWX@o~->lI0moB3HS$`gCCmK z@c#clUV?+`+^+7ww+AHYUObM)nK&7j;v9SrCr#1*$zWASO zI1X>biTDgI!mVcU{@&9)6E@;yI0awE1-RF2?W^!4?8f_W@C@DePE|h<_s5xdDK5Zn z?3JQ@@AtJ&$Fp!bF2c2I3NFuYw(L{ z>W9tIy}8(oPvC0Ya=!XLAE-YB8}T~qz*lh%?z}+zAdB{sa2$Skq4Es878hdgbp85b zovVEiF2Q|pHO{WH=hvT*dFr3Xxj5z{>Hh-x83m+@D;VC#gP5D7wg1anIzY=f2 zKJ(RYo}oMv&&3JYiJdt36ZNAOXulMv;-;S}&%p`Ufv;e{h1$1WtbQyWj??gmxCEDA z?{w{-w`yO6XJN08l;`76{148@Lzigp`?30V9E~4Zsyq$%!-aT0cHm<;WRdQLEz>?3 zpT{=5DpR@Zzy5MLC`0=PR>&rtjWh7DmCAE)V3wTu2_Fy6!(XmaUWuo!mP0>PKkhR* z5g*K!ZTQ$4xfm~6E4%TE&*i|yy7yU*9D^sVlT-1l>t!o`d4pVlpUahlth#p_N8#T# zDo?`Sd?9DzFTa$Fannt54L*aDmgxSP&C2s|>=wBK*Uyu^m#TjNdo7dGw5$vC(=N&qrJOc0gS~lb6 z#c~$@8JFS>-zbk>rF)YP$!WOzVYvvG;Tk-mM0wa}+8d9^33%vH_PFaYxduO3DhFk2 ze;dc)dMA_@S^ZLc7W-|{KK!EcIQ#|9z%NuP zx8ZHr{$GAcxp$uK<>PSN;UwfEny`%`cv{t72x@88rfz!vPr9jlaw=Ih=AzsnY!hx72l ztIDfzr$1!B9opZ)!Ps7{JYc8ttZQ-@o_Sq1+f4&~JNBxioWD~xIt$24mAC!~u25i``enMmA(Kw`uoQ!L64nF9kybOQ*knF;V zO=X|2bpLEKIU4`yE1U4{hvj6vxVdb@qaTr-c%q-|U8MU{TgYMfhn8{z-qlLZ#0&go zzXRHv9+hM9OReQ}{39;Hscn>3ey#o70NMAT{7GBcj4kctY@GU-T#n-dWv^oGr#~(y z;^ZgfES%6@F2dnKvJ3k>DW`m+d%xiVeCR3V)p+&OvhN}F6NBY2{M<8g9B%ZioQm(@ z9K7;5k}A_V=*o`^tG<&vaDob3|T=!}0&H z1;5ux{Stf?*WhlQl?NWxy;InTzMZKy2zo}hK;xeTktDg)wklM zI1e`tS6+;*xEiA2MZ^^5Rm zT!nYwkaOB!$41=k6`qG@;(+hfFUQ&Voq_6CJCtw2{y)jXVwAh_G93J~^45ct$K%g& zGVb}R@_hU)wqx^PSe;R+m3ss5lL%46~OI28{Zsyq*$#KrifVak0k z>E1~khTk$O&&9`a$$$B91{o_9`0NV836L-^7u>$$|0OTkteog1^Py zRq78Kp?(qGj=g?Y-s)}Tp*Rtz;H$U@$Bk6K3a`VyS9PzWNqH15z;QTul=2LG4%cGy zXyt)_=w3CB!q1OUo`mON3ogOgxJ!ci`FIa@;GSc7UbUV#2gl$G*o56U1qYAQ-s_t7 zMjVALxEycAzSq_N0cYa|@919e4doqi0Um)PYm|S4%kUl?_owp9*om8(bwBBU%7@{6 zyb!zb2^?`#eK$_W-N)-*8Qy>`wd((hEAjL1^88!MH{nbiJb`<71a{$_IP5R&J0+^0 zh__-ZcH%PJW}^Cr+uFZ@gYZfmjZ1JmzKgSP=p^1Ro{zmTl>eRsqgoX zyc&n&N0XH&;Ym0T=izev19swO)3p!$SNHp3BVK@W@ljlbUwBXZ?0ed$U?)C=6ATUR ze?ICrL;VuG9J}#J9R7g%4N}xk#8EgO=i!8U>eu3GJb9+}zV($~$I-aYEahob&+8y~@e4V6c|uYMd}g){MaT!_8qs9%O( z!rqN^F9}EB9XJiw;8Gm-f$n*}kFDpGgS{S<+gOwbWMEkxt4o}00cr&))OW1+iXXsvtpYDyu zN%#Z~YN5ROC+g?pPp}>Pe5%~1rTU|AH2xeX;xcT(^%iSigu`(So{1w{>3$(j!+}=q z!~K<;aVh=DYy`T{7-^K}e4YuJ1IqJuEP=64v#$`C@dFA!isc*vJI1|sn75E~~ zctQK8*K6+=Du01vumh*#<{Q*6!b7m3qxSo70e&D?{fJJ=EjStP!TGovJMnWHwGZyB z{W9#p4jdGw{H-t4kH;Ia1K-9GFRDM|OZAiSHk^;IU?*;4Q$Mha_Cv4{TX7b4;h3)K zM{d&Ijt^p=aOJmfG=6Tg`lC_Uti^|cB>zY2jMil5trhp_NbrQPy2<~x4-<@UY?I9;WT^z7ve_> z)pz5iI6qqV{=`uOc-}sqkGJ4_dd2{93tRjP^M=5;r}lJP{Y+92`@uybL?A8xQsGNcWj>(~e)jy2m@v>6o$=HRn@WSKD zi*Y4(;?NVy-S|@+7^~+!^sRCuUX0E71kS+C%G58!MqGyXV#5&Kf90h5W_%cD;tt<& z58H7SwwzKPF;w>+DVH;G9xlbbPb+uebGQZ%J)=BmnC{)f>3Hc`K>MtlaZ`e?2cJPQ{ZiD$m6K;6gmVQn~ZL9Q()X-i%AiJ>NIj^LqBOoQS{1 zY4|R-;ipc;>OqIpi$bN z!G)vcwKtUejg@0-*!uhz_|CAfXX}<*r;chpTTX8J7{iWR3EFZz4_~qNm&3IXzJ%4^N6>r8id>B_@2lgAU`?qi;ZsF2> z6P|!G@Hh8uFcjkN$rIkyy}&!3y}^)!yJN!y3(i81SKjzNWThSAkUek zyvtp^ze>Ch`%G5;@ZZX#@FzG4-^MmP(#<`55&I#zm4t*84oybxF78`yuA_LJ+YABC%MA~ty`&%!_6x4}?? zk2X;5oUeN~>g@UX;=e$?M{dGX8!FGhZ5nBxk)i(3I(zoTc!0O^z)zIFkBxW-PQjOO z^rz}S`k?mdcrY%-3$P3Cz`l#M|M9--z8@aGNgixf-lMVSzQK@yQ}5ei$Xcqr9NTgG zCVD@<%aogN82$*S;JAe^7w>zS(jbuEd9N z-Wufx+NvM*x!k9noPt|CCR_1EY{Mf0l{;|j$7Syv-OIwkxZM-Vqwz;L1=r$S9MN9= zLY#xE@Gb1d#vt{J*Xj8uaW!7`r1Gfs%C9{oS9~E4d|LMYQeJ>V@aw_s@o}7tS3jdX z8~1)zF2-Nr3OxBae%;zSlu{;U@KuJ};N!RWHa@I5AZA+pPX~ zI2^y;QF$!BjLkTylkyDg!iE3kot0PPt2lUzp0_4Uc_i-mq8x`O;#7PCTk*jz>f3PF zuCg6(!+v>sURJpBSlqCiY{iSQ4Uc?Dc{MJ?zFT!~V0Y!g_)DCEpNUYOjn`nGZQ9?( zp6@&Dc}?q~ek}eTC*wgqmFMEFFUvLf2oBh;d&44?oAGX(i(BN&$ z^BRG@zLH~yD^JGXVH=KqO}PVa#BRLmb>+1ObgxyM?EkgAq|Tmy|ApiFZzxYYs5}Yh zMt#-@r+Dh*|q={2eaGq2rZ1@m3sjLiZcKt2_lS!#Vhg3CaubY+Q+Ja4qha zsD9wLx_=PI;P8pc)9@)AT&8~9B;~RAH*ChKla*WX54ZrwCn>MO>#)yB-9L=O@fB>w zE}V&*Owqj>JOu}Qr+epdG+r`Q{bW3Fnw*FGC(FiD+F!>OynMRyeBA3j*{fWA3l7Ey za13^16CN`|`%HWnyYY+^<-w z2jd)^fGcnoZe>xw2=~S1_9)`2=GMtZ_Em7ZrKgI#S=zhvl<#G7cWpXzD5f|X8 znaa!X864o$y}0GdBXK#-#~oKFFUQ-k8@E`gJn)L{S#d0$lchWx`>v8p@p|m{EAMBu z@;F?Iv+>l=lvm(@Y&rZl?O(tNxHm4tHtfPp*JvM9rF|)m#q-uGFT|mr%TD|$_WfP^ zUvLB-nxnoM7voGkX`S*?{4cJ>pRQLPbXE8N!Lc}TgYr~-8(Z=4T;(>r8QXE2jmoR= z$JmWs*!K@Tf6f=`2jj+H%5iuN&d056$|I}UV-q%PQl5q5a2`H_OYrQ?>K9$pJr}OQ zH?}CxxUM`tk3PPHYjFRq%KdMse-X#v8QYX6VkgeRqqZwA#5-`=|B1T;xH{+m|Nn=D z5JJfGnb0O=YHM5DLfhKbEZec#gyz)hY(=ZoX={WKLI@#*5JCtc7D6V35JJd=5DOv1 ze(&o%A8mDQK0e>i_j~*Q&+Yd5*!_CF-mmk1U)Oc6bDh&?4d>J=KZL)=;r)8Pv`3V8 z;#ORY%O6#~67TVt?DvDl&&N^t4V;B1H>kb_|A0I2gO4lk_)-1#c|s1yYjHAu6L;X9 zpH#i`CyjUFBHW0}aUV_^Q2m-ljjzOy_5T{j<2g+lpMrC61-=FM;V-afQ2nMoqw%4?$t!UTz7D70=W!w4 z{aKCo`Ca4B#3lG)Z2jJT>k;sr>O20Buf~J82S*Mm?`~Fo7XA#E;{%^pz73y*9e-+k zIS#?k;wZfM1&uGk_uy81$cxJN;A^q}U+TY8i}JC!5PR^OxDLm>r1~Lz6;9u*em~(3 z9I`?6zK)%}`!5T}W5>(N=i*~5)8efFl@FwioN%g0-sXhe1fgAAn*U9g! z`pa>^F7obgC?Ah+#)bH8+<QfwQ@_JIl#ji z$GoTd06*2=h@YFx%kBQHNF;~i`(#ncmRKbgZ9w##dT_YI=%(>;~hR=J$tI& zjr;NUICwARogb<`1J_|6f8`JUNO|kuMX(+P*k^BfOqcTEI1wk|&v6l+^s(ye@Oj48 z_iLT_dhD~0`agtY@yj?DZ^X5D&rj5^EI{KI7?0bb*wHmpPQ$*@ayd@KZ{Z<4=2MLi zJy!LHcgq7O$ZAi`l`F+&v8es zJn0+dYOjj^Ctwr}@1`)?<9XT=;_=@q_H~*~Ob*g)49+uEn3?9z5Yk zjSt>Q<6XEOHw=4c<-fr_IOr#x??JmLKh@Ye-z)K=pXGY&{#6b+Q1u0avJ?CMCI^jG zJ{fmo-`|z*2vYt8?E5eINgRYf$KiOdKQulTUyn2J-9yS(;K%=zoAC>O$$hwOvm7{1 z&;O!hSMU0rxL^ml2!DcW@s&F&-+`x(ksSxC-$opUOMR42!bj~Sm*AIhJzlo6@?E$S z`v$B3skrE8h_Y6^uxd6 zWc=v9$`|8R0dfNl+E4DqMK~x_{kw1^uG?Srarif!fiFEk`3n3hZpDiNmG?P9{XfPr zxcWfl8*uDcxf36A5aW+zemD_75~O??&ia>Jj}IRw_u!2<@F?}W{9xtd@uXn67~hB+ z@DT3CZylm~pD^{i?oc@p_u(jf_F>AW<5zGcJ~2f34%~(Pk5>QN$17ig=S+}m@x;R! zk3Yg86E(g*RQY6l<`Hr+o^YhxfIq{%_^P9n4?0HuW`@a5{5~$h`9~|?h`%Y88wYU<$kGt?MI3!B-`^;CrBs>w9;CZ+K zyKpxy#{SdQuX=&{h2rNH%H=02e_g!Xgjby?_u{BUa=;?h@4Q%!!EfO#e3Mi8N__fB zatjV#BKPA@anNG*za>HWIGmU$d+=^cUT(%yYNrgcbW2EELT1TUwgWog-cv=6;4Q%8}K1%a(}Y=1*OYD%jE;yava`g zrR>4K<2wBFD&;%z#~HHk>FVd7DOck(9O6>`{4Tjzax#uyE0<#5Lb(|)#eMir9GI(q zn{X^X=xmM8#j)qeRru5*xfQQIS02D?&y&N>QvZ49%Sm{jV!0H%a09Nx-S{``m#2O^ zU8wOP*#9EABVYMS9J)q++}Qg3pM*Pc+*;)~<3hZDiTage$Hj6j{u=wO)A$XSFduy9 zrE=ie%HL5chvVBXll{+8{_xA?lyhYluEy8nZrq0b&r|)cI20dph5E(eR9uMH<1+je zZpQnTX?!<60f(Hg=PSg~_}QyepNMz7M(({*^~YT+2V5n;g6po9Ctjy~Gd`n2?!)(7 zFNc(?e!~rNG9G`UT#9E`$_+UFCb=6Y+$;xPqkae9BFExhT!iOWDPN0gaR>ee*IcW9 zQ*Tv$3(mVu9>Qlh9AL=_>DW1@4=mS%KjDV_vu}79Dbli_F&K5 zat*#07hkXOFW#ekCH~}I*{@Rhm9=v6&GH5J$sT;m{c=l{@`pYkmsQL0xDmUJt=DG{ zzKnd=9jd>R`~YqwUwNnU8*w{+u1?R_i*I>QuDwh3A3P*CY;M4}W=04RA;&y!X+1CguC^OE~ah^;`6e@=n}= zOL6$K$~WMN&&h50sAjqR5%nAQyj+X@Uyywpln;7Q4#tPK$nB3SzxPXWFa90ZJgNMC zFDu`Kk7<>Qo>smB*W%xB2R{83)jQU!{zV*yUwBpdSls-Y9MYuv9p8|n@$7ba;8o>I z-jah~lQ+CA$7A0Pxey=!u3V3oyeIeKdGE^sZR$6zQ;xyoKadOY8@L90K2*LPAMlYp zgiq;`Ltj__@{i>deAg#(CBE@fxdoSY%l&xGXL9fx>Q}c>j>4aPF8g;VzqCh=#;@QE zeC!v>SKx%pFJQu&ENC2 z9CI5x;Uk{SUkNz&V{h<8Kf5?M)?;$zsC*@b*Vtm4%%GcpZf5|<#9|sL+ z`~#bnPsD?`7*{xU^M2o3hmYPt?!+%*-=Edbe@Epb@p7Dk=Z;an5|8nb+p!Z5;oWzl z-!JN)gj4X{xD3zQS@n(hL)?RNcTqmzSM|GXS2+y7w!56Mhp+ee-RUP+;B|Y*&GK%?Y9PV|oBWhpe*O5=cci=+&JTAlCxDxNP*KXeX2j4*TI}xXk zl}nASpI5~GOMaMqF8+{wQi$?D;xZiJukn84mCwUTxD=P*YTS>XFt+Y*f+lGEZ{$;O z@ZRcQ7^-|SuEXcyPF#(Bj!^ya`)Irq`|K-M;c(oFm*XKkghP*1zvlxqJ`w+kJ^0}L zl<&Z^u-{P{zXnI)ko{F(f=|Kq_N%&Q1;-I1Lf$6%3p^gkCA=GDxZPR z!8Q0t+=_<|Qhj2$#-9}=7vX|`$sPC`+>c|&Deo7d@$(LrGjLq6+=rjWfs<7KCyvGQ z4^jOPF2%8vRsS3I;I)UUz7&6t<04i6#9`#|u_1C3ei|oEQGMiiT4oab~FUo%nMcHBJ39k5E1yk3Ui_!}s87Jmo0mTk!3;3;TvC9~Pzlm*FbB z*U`$i;`4DI_M51D=yZ+02Pfjaj!`}XSKXUFIF2pI5 zlyAj7IDV%3ojO_h3j7JK!DmD&--&y$BbxD3ln=#U;Ch@nRrwbD8}7&H)07W9R{iR5 zEFQqQI3`NTRC00(tHMj!LnyY*xK5L#FJ6GfV=F3@lYMk7H-@@Jaqy@?k;dLj- zN%Pe2j)ig;o*pmz&R4!2M`6#2%E#lq7s=M&tF#`QaXhYDtb8$E?34o+sD2af!Y`ks z{2;!5i5zl*>fH%)5}uMM7vYV#3SYQX`5`=Une4w%{rpdso%l(djZYav%Nz2cD?$p5@Ai;loaslkj`E2Cs1`--dheAYPK9 ze9$8G-yv0w#&6;>yl#c^{)<(=Uz(hRpTimWtaRn8@ZN5@1wV-U@aZd+_c+zxca>a< zpT!M$afb3;_!;bblExpEseB~96&GRuGnB8wSKwy+C+@|koT+-hCF=hq4#p>DDW8b< zT`f1^=Ws7h&Zb|2`n`%{aa4}-9(;PPT!w$aZTQf$l<&cdu}`ApS@O&!h09W>G)Axj$76#-;CclTkge&pCkJ%Q~zRP>;63) zKTp02zgndFkdswkajqPPm!BtR;R)xDaO|4uV#E1 z?!xz$DL;f;u9O2#)AN6JmE3~E%H=`4_8K`bS@qXmD<|Psuaiq~Y=vBnqpz2nahlf- zN8EDtt2VZdfA#6|6XauEau04ukppf}zxEaKdSh$+5PsL2cQ^{ulplMeRd1dzU3TJP zw_J*oSIQ6KYTSW)@P3tg{*WToFEh5DKMt=ZpMo#LxwxMCD*Q7Jy+ZwF-lTp$SIQUQ z{;TDejIHB6P$3W8ESFv{b@@U^$9pY`|HtVf-(wLijdkiR5fc%yvOZOT{QB(LwM|JkCOL& zTKzu2QTU*H)GuVc@=I_)lYG0e)vxy%xs81Ev+^GIDj(b;Ps1^ICAR*aiuJew*W&d! z{w3wRanc5P=UVkI#K+)ryaYGl0^E!5#(^)ZUk8rEKKIcdAB=18V%&lAaAd3cU5``n zTeuK!#x;22{d&F*yd3+yqJCvK62FQw@C^@Wd==hMC%52VaNw)z7x|#_;W+;xxfnM- zEO+3I^|H@v8bAI~ITEjYOwPr14RTSN>fe1_uE(c7A-Cddo|JuFSAANe9E2Z!N_OH; zpOy>p(0aKMAJ-&z;dMCU4fTKN8RbjyX5539KdZd;_h_ugUAPi|gX?j`bE-tlI3ve&KmHME!H2yVg{k+waO z$Yr0Y{$(7qQQq*W@^$#RZrR8Dy%p=x{Fxku+cwG-c=qRV2Yvzv^l1DMJ<7-8a-58B z|3dlnFI2zNmvRaI3m1Q>{CQs~Uye877M#+n`~bcR`+lYIuV8#;LmX)Uigjj zrT8&iizj@md><~yKD~OrL0pO__o==CH{yQ$Igb2V<1@cgeJ=L>UM|76;z}I9N%;=k zjve2q-?jb9N8o{$v+UQW@mJs^JmnYV zEAXed2haFbdEf6e{(2mSU&FC@he6dR-`|`Cyig{%@;f3@U1urzk)q@2-o7M z&FUXLpnh36>lgVEug%}D`&B;KvAg&F&VNu&-9e7RV|KJ`exE2l)@$?m;5XIpHb(i7 z-{o0e7dyg+>={A76rj-4u3;`V9sCa=xs*BP?kUg}pfQ_jKB(Q>=7 z)xRr74)a%id8}MvY~{1&%Ab?>ohOIyt$cKxT!Opt3&z%Zd`?jQPx8GuY9EcSU8uYp zcjJ3-S-kQe;sLzNz8W8RqViL4G+u#|@%6X}x8i2JTY%FO5{EJxwRxEx<)?EQR=ryL@OxHLZgP&o;&IZQ6Y zSBA*VIC{LEuM3}y2k=^)oTC0$Vn?d{EN;g?;+}Nn_ne^T^LNV=jIH%X<5XO;Qu%YJ zZ^n-uuJMVhl;0F8hi1s`Bjuzs+n^$9lwNo@s8oDcVw&I3>=B`a0Wg!LiJVnZQPFcnWTI_cH$88_aLmtRX7R9 zPF8&x{v9{tm`LTjaV`$dRllooB7O*$;qRtse9Bp>Uo}lG_sB2dCVXj>@{t9~e>7cg zz$r6i#~S4in<+=*FL4DvHCp*bd>?MdzhR%X>bKxn)kopIkCU_T2e=wX9It#cJ|RZ- zFH}F*EV%+-JzGv%r~L18>^xKT6?h|V#}{QO z?{|&Hhpm=_@jEyYpPQ|G3U=qnRrojTd#(E2nyY*ycAX{1;dArkHhfRM+=HikwYE0{5>D*k?+kHJKD|P^RXVcQ14SCzl_W9m$)Av zP^Ny>cdI_t*m}N3T!zE$QT}|CuBWt!1cHV zN8PXdW*qmBJmD(!FTn}88=r%N9#;Lm*n>N9Bi`$3_4BD$eVnnie|;a7({M1p2$$kp zhsQsr`j>DF{t1`jf0gU`+VFhr-=OgxW9#|r@Kxmf9#?)nuEjqMkAFh>qpo59<0RaS zFUG!4s{Uadhd;ojc&}^KuOH9HEsYv~J`Q_Iz8`z=`#AJz<$bPGzg!%KgVrm*47cK9 z?AxULgE$tyi%an?73$xCr(*wSG(H8#;|g4kU&d{C^YHj*HU8-9)jtEDifiy?xEnX( zpyxFHOPqxFy+Qpd@I2g(&&Gbu8h@{`b$&R^-(#{KZ{kS&J@r+1{EhT)QN7F98sCi{ zBp=tR{P(yVA5^LODD(G{tjBDeflo2Et{26)1V_HA`WCOv_dBo2Uz4xI0XM0CdzOsQfGgjZKgEL|$-CXGe*WF^6r9^5r{L0F`3hXzCqID)H_2b% z@_sqs7WI$*L7st~I1{Ji>u@RFfa~yP9P*?3O{`M?N_?`{=I7(TDPN02f0sX`J`wMD zt2MsZ;k)y0-s9)S@xF2eZpF{ze%yoOc2&LOHua0%O+E>i<18GyyYiQMZN84dkK-Er zBlQkH)d$~Bf9%AG_*`6!AHajS6Nm4?e5-YS72}1t7MElHJym~?v9-S%@R&Q~4u9q2 zaM9lKeYgR?VeEZ<;lIeo?xXsNcdB0!J^`0ux3M+81()HleKr0G>bvnE4hm3y#$D>4 zjI(eBuE(|bO`N`;#{Yop@ZL4**Nr2*b~ya@SN%!27N3PX@HLEYK0x(P;y(N_^?`xP z|A}MqzwWl4-~4+}c$U}2j*0_SpGLk3UyA$j{Wy87>R-p@_$S4ua610jYlmY12i|Y>Gyi|I!&RSv8*m}+#P}RSHqw&5Es9!yvZ)~04y&+KC;*XZAsIS5=Q{Rn$A>Tex^^@zg{=j481l)l0aUZ@3hlQ*DIh>CF#1;622i3m; zm*PSE4EB#uzwdE44t_}E<8X|zwf<(Dfd}yU*l&{h-Ng7(+>2}Rw1?Hd58r}=Cu{r= zuE8nws&B!MVZTV#2Rx#D5Iz?>aR)BMhd-+NYJ4>gouYpII2xb+nCjE7vTcjh5K;yv8rF!r20bq zJ}$!(o>AWSIMrW-!*LHz!Y4he`V!oQhw#zQDIa~j`W50_{4TD>@y)7lz~|#${3!N~ zQNKTNICeg-*H zuESNh2|tY6@C&#Lzl-~DFLo@^eEz`xc&`^V|6qI=4#(4Q3|@$x*o9MYKF-CL;u3rd zuEY=HTKpnz#_!_}{4MUon{h~j*0*nq)>DiR$K`k?j!9JgV(i3exCpPsW%vqQg>T1o z_%YmsU&d|tBiw~IVZWuC&zP4q|3G{I4#P*`D11DQ#R=GnSK)Mg4$j3_<05<)F2hga zD*PI*!=K_N{1a}&yKK;Uy757{A0LB#mTCQSZ~#6Thu|}DBt9R<;_Gl0z86>E^|%JV zi5u|exCQ@;JMr!>YdwAVU_68;6pN8vk4sOC1;Z}Sj?!*t^KKvYZB^?0vVt)~SahTHKp+=Un7KD-?JpQ7i_#liR@9FA|mN%%fojGJ&deiK*Y zjkq5Ff?M!zuV_7;cpUDh|JxD7YqZrp|ka1Zuv(|QJR z5cXZp@xj437DwVF9E-DXA}+@1xB^>$uhM$d;!@m%D{(un#XYzY58_tr_d3T12jd|e zi33j8^T**ZoQz{}E>6ZJI15+eB3y^da5JvL9k>qn;wC(V+pzx|T2B`a!F@Oi58-(1 z@6!5Ga4`1ZNL-3zaTQL)^*9~3U=QxZCAbe)V#k}Be=QEcjW`sy;u*LPpNRcaSPu@t z`8Wz+g5&W`I2qUBEZmHXa0f2Ky|@Yw;X3T!uJtwJ5ZsQVa5s*}{Wt{&rfPj29D_@- z6IbDMT#pNJ3ogT*xEA-}9_)Ba^B=$g*s((M3B>_88V6%14#Vj<8W-YtT!vF{HTK{J zT#Q?BIqt&MxF6SJpSL+4I1qQ>Fx-t}@BmK4jx?=51N-439Ei(tD6YX#xDm(UHk^dJ zaRwg1#n|^9t)~J9;d&g7n{g~|$4R&wXW@Qaj2-D(Zw2JASD3RN?(_Jr2d)cqSgei?HvR>Ys{(u?L6a zOK}Xo1v~M>I0e6mbMgDQ7=Me)@n&3u_x(ugYru!&7CaMo;>EZZr{O`o7W-yty;tBs zd^?W8kKqjbGA_g);ZnQ_SK={Untv@m05{^Ja2q}zci{xwhgabtd=B{I!3W_Adu^7Q9ynX#*xEe>|1{{aGa5C=0xp)YdVE;lre618@rt z!JRk~_u*LV_*(N%!~r-Rhhh(o#wFN^D{wll!G*X1m*G}ijXQAz?!~Qm5O-nUv$g(y z9Eg3s(RxC0AdbdiI3CB~WSod|aRx5NMYsZ&;~HFp8*n3T!)>@5cjJCMfF0*(eZJpn zJpnifhv9G>gJZE1C*c&Fg>!K+wtlaj^{BuVxE9yoXK(|48@J%Ea0ed3J$Uavt!DsF zz&=HK{+T!cJ8=kh<4Alqj>A{uBwT|t@Kd-5zk$o~=eQaV;s(6Ocbb0-J`{K3X?Os~ zW8ZVNzEm89*WfUG1&+eiI37QalklrJ9e;{FcmS8+UBB1-t=~&%J;vc0JQ+9OINXAl z;|`pMd+?=r09Rq3^Yna=-~hYhhV=Sw4O*DjAL;mPQ-CI11IAmoQunF39iDGxDMCh zCftnMa0l+fy|^C_VaJ79um6u)PXG?VQ8)_6;dq>cQ*bW!;1XPlYjG8B#Pzrpx8P3P ziTiLLc3h7Mw?zE|aUc%E*6%^J9x*ryC*nAqfs=3%&c)@p1lQmy+=v@+8*atjxDyZH ze(d`*$LC_rHwXvga2$hUaUxE_88{0U;$mEiD{v*Q#kIHzH{myM#wD6xH!j9M;tIUe zFIrDC4#e&FDD1dY{f@%{_#_;H({U7Di{tR+I0@f|Gw`Fh5O2Vx_(R-?zsIe3$6vLc zF1$bP!$;yFd>r;G)%up;Ane8-T!;to6*%fLjlT_-<415eZo$5ntNwi)gulVz_)i>< z{Rg$a6dZzcaTG4b@wft~;2P|~9k>+t;wn6d>#^Szntuxp!kst__u**l_)YVV#{oDQ zhvFJ`+dd^RN?NgHvz~&c#pSV*DyD#~e{^m_M|hcDz6C!AIaB z9E}66()ye@6sO}TT!`aw8BWI4xC}SoYTSzJaTjjE{kQ}B3~4<*I1mruFzj=+<{yIt za3T)D88{Lb;aFUb6LAer$BnoUx8YjcjT`X*Zo|HRYCT;z2>0P|JcMJhU%A$ogoAJv zwti2y^*9g5;c}dW@5EWS0TBm<9<97`&FplVjP6i za5!FzWAGK&iEqa#_z|3iU&O`uJzS2z#?^QTH{iW?(0W?%p|}H2!TmT6JFeIIPQw8> z8;9Tva3rq4vG`t`h@Zyk_;u{TpWzbxGp@k9?x^+D;D6yp9D!T$T-=FI!M%7j9>f=5 z-y5|4>u?CZ+j!j0#g3G(m2bis*msQb5934di}-AOz&DJ?iFhL}z`OXU-}Sf!pZKlD zhwP+$ExrQ#^(kMC_rO2mQ}NzAYrF?{;7Q+U{H$G+zZqYHzrYXSKXHz)>Yw^v<3Gf~ zo8(P66wlmM^*J~Zm*D&HyZCkdGd^cGjlZp5{h!3|;$wGLe%TMom*W~-kDKvneyU&j zqw4qFLq6~)c?O<|+i)vhyr=5p22}6HPJ9kd!sR#x-;FcyGdLG_81K2W`TMfCY%e+N zXFcDJ{(AmuoQ!|Q7vZtLsJ;eI#mn~A_%HGP`^b0ws``2yH7L*6SNUd~j9$72@DQTS0DgHLfPABP<$$tk!RXW*D6%ID&ZxDb~lC|``nCCa6^5m(}~ zmnvV4qnF9G*m1I4k6UmfE>2Rud3gLOaz8%qG@&1K!FFs(M+>Zm#mIv`c=g5xD+MnZ!WM6#Xxw1bFI8P47Q_q(}aZ<4yj*Bjk zt>4>iJ*qF1V{p?&avbg~k&AK9#d0a`zeFy_e_bk9V!zAeYP`?oaxD(LLJoE8?w!vE zI2_+xrhF7$ex)3PgRhd~@JHB*Z@60d6dYeJXW$*Lk#q6WxDe-Et9&s&^g6i|zm2Q$ z9Tm#g;@s=ydOYg}xe@Pkquh)?!mYTbQu#rA(oM2s2ko!FurGe_X65~H@-1>89#bU; z*Os%0ns3n$_C?ohrSSKcW%;#2REoAJ0BxfO53?fAC4 zmG2xLe~;|DqxMHqt?ZAN-7g0YyG{-s_CsD#+ANRCehzD^ozJI;)r8v1suEvee$hA1&Ik_I^ zHOq~-{du_=Pk2#o#YHW0oR8*z#7nXhzlW3X#T%4Q!I3Y^8Tccdi!W?dz8IhOid;H8 z{#Cgg@AaBoiNC?s_?b54YjOGOawERy4Y?UNy(zcij&`{n_q`=|;-R4oaj_2U;D^#D1%55HRdkH*&T$+RBd<7)iyPpWUkV+Z6`yb?Rg zHU3c?h+o5nxCfWxoqkroT08-_<09;PjrzZhgYnO}7zh4hKChc!omYv&jji*q6@P+* zuT{UXzbYS&TX7ur8C1R+UykeX$G8`V|E7A!b?WcLDL5Aw;)8xyeLLQOd+~XHC?8Xy zes^OhUOuFJ1}?$Pc>h1?k8d)z_D?rH=`T6zdiCFRv+Ts@8(aG?1%H4m@npvy-h4ei z75m?yeis;9&liesBA+|V*EXB#yjlH_~$i#ERMn9#^ZJ~|NaDS#f5mq zE*f8q3$XQjkF3XK*ztmVJNCsbI2ZTgYP^T9`q$#gxD_YiUVJY0eNoR>gG2FKI1c}c zJMqE0>iI%jG=3hg#@RULCFQGdJ#NF!4ayG~Tl=pZ@435Ni;wh^>+u5p^qA$zI59-m}9Zg=zhoAEoi7oY5J>ix`tYE{1;?2B{uHu*gqj$mAh>t9j* z&A1iU;com4?#1oc@v6pm;XwQ?4#qp|qvsFBd*B#+0M5lB_$++jz8YVM{R8A;JPz05 zefCqn9#1zOx2O4i)%Xhnz0B_eE8E9>iEADQdR^@B`@#GY zF!Pv+yS|r`abur+E)Mxoz71D0{yCh!+&qbSbeq`A{65KnO{(9)Y_?oF(aJg;hvE$K zb8yLbs!zwo{qn`w$^7oYL5J<+?e{Y7c~AAdcp%-IcI;$!fc5-UcgShDr$a8oMb+|y zIE?wef;Xwt2nHui4yb^OrXNY_oMNMy~fTn^Q%NjA^2d5O)dY%aF>CYv9% z*}AP9IiD_@zqi?8?$t)tkF|M*%}Z=fwRw%r=IiiL{ms{{qx_)F&)NKj&F1TvQRB_W z$x$}9Yoi=u^JJT6*}Tl=vuwW5<_eo@ZGOt;*KPj9=6;+1uz4SI?>O>!9cuGrn`3NV zWOK63c{X2UbEVA>*u386*KPjL=AUfdb+6IuJJ{xEo0Dul%jOGguCVzYn;)_H1)JZO z&AXHy-`VWr@16ga{6X@T`?aHOj<(rp^KzTBZ9dQDN}FrtE%RxV%_*eECpPcAw|D+q z@-uACv-y6TzqEPxeMa}Y)aE;F-emI$`;H!;W%KVguMHSof3?kT+3dgH==!BL-)!@z zHjmwZ^mwPuM;tIZUuScN%>{v@>)*6_viZPg4$hY2bFeoH>h<|3PKuz8cszCok=jko!Do0DxWwD}#IH`(m>uhGvp!R8p7m)m@v&9~e9 zw9UJW8~uC58C{K&0RL{eAwvcJ3!vS!DEswztrY4 zY%aF>2Ak__e$nQSZ2sBieM3giH^k+Q37v(395?p@!O`U7k}+~$ckN7_8o=Gitcuz88iNjAG|US)HR&1-Ew-{w-A z%Wb~F=G$cRsa}sdTmDg-pR)NCo8Py&&*ooj{>$c_Lq{L4J#F6K=7VhxwK?4888#nh z-v5p_VZD#eGBMjktckfM=9!ppBF@AD6DOF6H*ungMJ5)TaGE&D#1az;CK637HL=XZ z$tIFaoMPfs6Q`NTG;xNBGfiZfSZyNPM2?BGOyrr!H{mf+U}BAlwI&KptTS=8iE~U8 znK;*k_5Oaoi3?0zXyPIhB_=L5afyji6PKB|+{6_o%1m5o;wlr@n7GbFg^BA;++gBH z6O|@zGI6tsTTI+$!n$v$HgSiEJ5AhWqQ=DCChjqDuZda{_nElg!~-VkOgw1fArlXq zs5kM5iAPO5ZsG|OPnu{n@stVcUSz$ACKJz?c-F*oCYnvWVB$p+EhaXYc-h1&CSEo1 znu#_OubX(o#9JobHu0{B_e^w}_`t+RCb~`Zniyx|U=zV64l!}4iNj2Um>6$jf{DXT zgqk?Q#MdT{G;x%PFcaUH_|`<9iSJB&Z(@^)eiJ{K_|e2qCI(FWY~pAWznJ*d#6%N= zCVn&VyNP2=Y&PMr?xjqGn?FaGm}FwIiAWPuOiVQ~#^iT4Vcn8^CeAleY~lhF7n-=pM2U%uO_)cq6`z@7u+hZlCVEVK zVd6^@UzxDZE9*S@pPn1T=f%i#V&wS{XP$rLarZU;cgNn}2r%*g<#`Zp zp3gcLCYi9G567BqF(zi2m~CQ?i3KK3FtN}?yonP{Sm(%pa(<+ky5ZOP`)1!;uW#!d zOE6)bW7hez!bF;h|BLhJe{&95=isqse#eTyEkD6J;i@ zG;x)Qt4)-fxW>e_CjS3=AA8I^cY}$?O*~=ZKYc%Y-qgNe;zbiJCN`LO*+i>}S4_NW z;x!X(CSEu3hKV;#yk+8T6CEao-`74gf41J+x=eg*;u8~}n)si-#~o?L9AzTR#L*@u znmES9zk5HLZqDf$CT5z5HgTMZ<4xGF`@s&EbKQdMOphx&)swl#?OL7VcDV~Z4wviH zMUz)&6-K7zpOOot2sGcIA6gJ?^>AMUziV zjGq*4hFEDQ*#vLWHaBngsj==gnQ88b@YS9ySGxDl_GwIa=4GzVv`)#X`JOy?>S_uW z+E2>-%(LBplbe$6S(p2_$^4AezfDffUa`*O&i|XfQJEh1>g_-8c6VMx)GF;fYlpf< z?e>LH*{Rt%D>AdwBPQjp^JL^?yWDwsIeF7Pd8ujc6{%@wY7U7D7X3}-Tx$o9N>5u+ zkeTJlG@T05+_|32oNU!3{QDLo(|^}u*}rcwGQHgv&ULeMaWo12&Cnv>;D^-jaWyqvt% zsoq_>Rm+AeP2VM{nfdPYxx?qlaK^OXc3aMg?$tSY>xOF=DM@J=dhQ&g1vtp z-c|8A*{jUnwf^HBCN9m+&s>%5P9L?|+T+`7u&$3TJ))NsHmaQlK z*2>|_EL(%WHv0#wqBLq(|)+ho$b!cOf$Q|_Pq1AB!ig#zP@?^xBH`5V&a_dP*)e`rX({js}+>xtUl9`ray|1R`xve{n5xvZQ zwC#DeY%_ne-L}_;Z$NlXGh#&JW#;B3Coj!RY`YtEi(#dlc{#c6JkPoL#qqw_dkewbjDgYxuG(IH{F}U%%FWy;lpf`qVAQe#_Q`T(|eq;@!Xt zVAk;UX5>b$mTcN`1@YQ@UU|5q^lq{K$C~ES>`d#!i@BsOO`I1s;s$a!VO=4&-iVCc zy7gq(cI$Q;EwJ7qvb`5v@6GDSXIGAk)X2o}G`F5Ot~s1N)|D#HTu{xrd9yONICBei zbv4JrdkvYDm6_^2UZXast82EoYM<%K@!nFq%xliPg_sWtqqe4J=I7>E?~txV3Fc-X z&3cD3Z*Xa6daKOlEmui%CG#GEty_jUJW(@Cp4hq0Tn2e{ntIo~)U15dHm|^XW3)Cb z%E`9bW8QeP+^gMYC#H|Av@Q)>yfk~Q`IKbc5f)fC-{$LyNsAJunNNvp%tz3u1oLih zz5F7=^Ic}odEffydaJCDrrVZ9n6fnM1}NK;|8MFhO*VC@>FK&fH`9wVpWw30Tl>Eo zGtG?2H}^Z%CrDS8_j=|^wO(K5cK<&Zx4lo%|8Cr5Yc1LCwYu9qIn$G&Yp5&7dpXbl zcVnlUp_(BR8~(;8BHY?LnX3wN3i9VJS-2=6+@&vp=B`+j;F;`lMNC_hnwOcH?OAZb zqJ&7-^wl}pIr-iX{8sH$bJ(oU?AD(nrs?)QKF3@eM{MUyfVt+O12z2IeB0zUxB7el zw|(Z1GJDr$eHrELKizw>rJ933l^cF$7`3t>JI%VmUNCP>!erOqJkNCR2hdn|e%hP} z%{L-pwP~qW<$}!k)e+%lyLZi7pK5{m;$^P4d`hgxx=G*O61KLEjoAK5{#Q@A$Slg7 z?-}MjF5Q*u$y;FU?XBONc;`3SHFb^mg|Q$Z!o18QUDkiCsc&z-%x3EpwdV+u23i|1u@qI{JKx;#}zc>S=hMndU?EsP8Se-(j+OMZ3&LE%Q+@HE*4c zqW2q4`(D}p@F~^=GPBKBT`r9p_4@b^=P=c}`lfo)GF%?-hui$Q-jzgJw-|HHar%!I z;{8%Cr@%8}DkEO0|KUuguQJO?)3<#ta~ooP-M7uzsy(sY(vh$C?ag}2mwndroN3-B z-C5o%iuV;2F=<6=erB5aBGLMeXF;0xoQ_B{=d{aw1D5H@Uyx?K%qF|Go|M)W^S!8d zcTJ1WT;V;23A0_+QP_IATYsM7vF>>moZwuz#=OJ1qGHV@!+MvRyV^7`Zyx5wu+aNr zu!hcaEpU3P=5f`a;-u}pT)ewucwrIN!m^l%)#`t@GOc68izZ>ZYt6Q=SWSU1Prb(= zR$rvrj=|aPyqw`n_<#Evp5z*NwV(8VdbQtr!HSs7sqFn`R&)05%Hd1ch;w^Uf|aiU00tBG;@= z{coP<9~ZJ^XL2!}mY2HLdRMVu3byX`t$*64#*<^cqIC!0J%l6ApNRj9^Jig#Ct`A* zzHrl5=8KFn5VKX0qN$p(|imtCt8AMx^+`v-XvD$3=f~a<=gm?6E&-d$jVt| zu7Gaq8^<}5=9;JU4%s&IZ!Rs?+w(T-GB*QT4yrDi!;L3#SL%JvEpNQm9^EoY>(HC~ z0kamJirXarepqV4=wYLdlIb~Og`q|@Nw#Oiuu0)7QmusbV-9nh&L;x1R{h{D!L$8>5gl|YyUgf9 zk1N;wT+E!B9LoRxET8H!r)Q?eIvW26%~HMJZjLy#|BH!j?e%{!vCQmz^Igka>y`Y! z-er0{Oc{Pb{P#=T=GfYAg0}Z^w!fP7G8;XcDK2g&t;?_Z#699H^Ul_uw@)O>eCkU# zKXo-fhV*{CGvD$@glFe?GFPs1nQO_GQ`XyTJH?S}%{|-PC5sZw0t&J-&DUBk>w5_E z$;35fwfXUk`H8pQq`5wCr-gk=+r8pOx?Ft7cUjlr(ev8gh}f+!R`1+Rjqbax`I#Rn zc|T1KzaXuON#Wj#oVBie?~i44eV&^+`aII2BCR!!nuE4aEy(_xjBSxq(=zhR=jVt? z<_AEj>FZo+*09Oh=EqATr!-YFjy%KssBwDZ||P7GUjV7vv=2S&1~~lX6+s8E!&;9I@A2$ zVBFrJ=11yVzINDpC+R(VO^Wk(d#4&D=2;^0QnOc?|2vI-mT3Ky-fWzdp0hgD`r-WO zts~!&t@)_d+z`8RR=T`>wrm~!?zLreWDY-*w6)&my=i#F_TP7gvs(_*@P@4iW6PE; zN5Z>h*`f&ZqDd<(OkI(=CL&^*`Lp$HziZ2_koJtN%Tm31e;{DcR1ae}1PI;2fYEXe^{CJ9B+AV~OnL?0aWMAwh7y zm?+Yte-rt@qIe`hyr2YL^zYSo#jyBS@s||hg`SJHdkXVLv!`#n{WnOpErO=%2S27H zz#;2UzQfT5eBn2c%oJND>`kNl_!0DpPtKm~kLEE-R~{*Upg)34ru@JIQijEKAD^dN zm_GMNx!Z4=AI<*roKV76FQ3@QZV%Y$S;UjKq^HPDUG^}m$N0-qknHu zOM!nc!MEx^5LUv!;ojkYpI7SNh~LkmhXkANR`!gz6;uv>f`8L-@bCGv{x=9%{e5MBk$w^%`oHKUz2{4|T=ehj z2b37hsd^AzO3Gp;WQY&s0!EH+HlRXg-I48Wq3YnY;(oo0RP>O*e~jl| z38dvv@qbCI7+~=@RVQ-ASPdSg>JRSJk%L$(K@3v=IV1ykk5s}2f(^Y3IEbZp&XBi& zwyljjVDlBx*So`^PYHSM0|leb{dEW8@&XTmOL%=kUkakge?Ci%=CmhPks%GY{MKSt ztkTbv6`e({;&Rk_{V1YLU|!V;-@2Fc27I;=1oSh|E>2nNbT$=QFC&E*#Pz zzSdIYWlvX#(2-xK6A*=UjS9L*t!bNDm2^-b6HrC(Y_4m0usmD$7Th&~6|MtFP0oEa zo1(f5t{PiaQ(cqQhE(NRSV`Zz7C8G535aYFC_%&spn`vq6Q8rSC*T)Ft4$6n(6xJl z$M}sc&Es*sCg{2}yqB#>nNcnMUlh_zuaF@L+!jGX$iQv9+p#Y>NMZ7oY!AkyOo9|Y zUOdPsg`3{H^y?n7qa_54Ak~eSb&)9x;2EoYGDpfn;7*HCk4g@b#TiY*Gv z%bn%Bi5Q52X00_vJFdRt{CJt2k6it*wyWLA}ije9tI(7=OB_!Z&Ub z6oCLM&9@@SOcgRn!-|m#u4ND#WA#u9Q-vHw-3ywn(`mBB3TUiH8`qP^`FL7hqH&devnnG&(1dBb=h>#vA|p z2zv{GgP0Q2X~)=|vyt7si79cloc_oOy5`1imhI&E3&sPSK6+5QmQBb$qGD_2gAliu z$|lXrY!ym3!vc9;?CHA>%LnTD(jF1L3unRw9H<+^xYBmovf#T8S12h#&k&S7;;uWmiv!?3JDfm! z{@geXB>n>S-S~y72=z5oAjt+wE%jv|;~vy_iM!v^jNTl2M4EB1SDH7i=|R=cUH z^cM&w7oBe~Kn58_DXGPW%%oQKu*hfY4Rb}SMHo!VVT1}|@`4e)=|{J=1|lncs4bd2 zm+~shP9VmqJr6OGmCvj1$bjk+gp5*Q#uOwQM3}PDhMQ@!9fLjqABt>AhzvlWWmt$3 zGy}1XD#d9iDyU4;TqZ%(iEUAIzKD^)v+U#QSjw9w;ba2vAHL2OyDe^-ewO||*zM;d zc)+s-tVv~*jz(6~8^;?**~SX4l}7=ARh~vc`I1~4rkbvmQBJtvSFJ*u+zVmLcuTE` z<7bE?TxqI&<OPo`axdFaSN-3gF$Bqg;Bid>zJH73(;u&2c<+_mtW)#Ex)A4r*UyrwC|>VX&x*%i3UO zm`B&Yj{dOn6YWzJ^Sz~dr`s5@kzE`hLvd!ZNPy}J!yESBNIG^MY!1QA5e>8Q96|Vt zoVaRa;9aL)Uxp!)9XGJ;u&oTp6;jp}z#wpS1o}>#!LAPDRddIKhoA>3vX6KOCN*+; zqXSSA)$E$J)jOIU;_UnwebKJHApd&ZMk^6k-dkSndQ(aA^UDE#CsokOxRA zxwzF={E~i${wUZVqB@3hrSANKbEV?W*3Qao3~g}{ptFUer+9(Ly0Wz3>rz^=Xof-Z zk~NBdq&e!ik>ebjO@>B;azj#K-$$9%jsdwqHO0%gf$ccj=w)Os&5nrM8jW7-n2uC( zt?$phcP!G4<0?WhNfLG2M+H*#n@57+Yp$kEbp~FXO&+fcPs|~&6 z%S0wTut0Loo^i?}p^35dCMIv!V^6(JDAj}um3 z;sBy3Psn3el3Z1J0^!8NJhx!liD4k4UP_=6Tq)wXPg(l3WXq5$5ZzK$eS87Zb`>84 z(qdr+CX}dZXw-KAtfD}`Yky%Sd=j5$hwf%(@~y-&M)#Bl&va7R-zC9 zgwdrbJGuBi#l*D-8k_~-;-_gu9f<}wFhlCKgv0e!R-)y<%XQHB=9XqfkM7AY=WK5R zn2KJw_M_rx&?FbRIJ%Y)~KoL}1lPkp!i3d0nN6Af=4^<9qA#+s!s9xr) zzm(uF3c+}Vy6<)rd~@W(>L0}$;_96J96MQ?wyY~{NCOr}BS#j&7-r|zt1a~0x%#dl z=$Wuo#!S18^M0hqprO%x6!Aa#Mss{nF>Ohotty5jQWd5JJFlx-T8K~~P@of;Xi$~y zV|jo4jbxOWOoHib5eAsj+Pn)jA`OzeRWKkC!|EVQt#tKk|(t*T+;6&DY+X2iwR0DB<}PfG8@%QM%?Mg;TIj%X5D zT54Ovq$>j9iXIZ9dI@geq4Zkhty`*F1hGloD(ldvZ>yX?hth$naXL)kop#uSB2fe4 zVdOIt4*#9E;_yLk$T)MXg$ky<*qe8sJX6w zSi~*l9CyS}ysQMxPNuWd(B$OW@CPT|Ofj0PBW9VXIC0y}~NQwnk2p#M5I zNUJHb`FN<_Ap75fkzF)51UPWaUk-~VHP$=(2aWxn<88FR*Z^SBLWlRBuJ6=X!pRaB69Z-ny!H zcZa%hAUCbpbkN`xJ-`&rHqWZOTfr(lPP$Y&?od27a7Pv6@TDw_ysJNUvHC~H^1~Hg z9X0wUE78fQx;H@OC!XEI>tX14$H?+@c$s6dRL8o0Csk%cI*^(}^Mo}K8!pMmL4F~A z`26vo3+nb5Vfd-(WaS1ZNqbdrQkH6HwL0u*;AnxDObcdbt99K)fvuJl zMSh;T$#xs)2&#gHd+0;ElCTx$E6LD`Hs?U?sq%=DS#9Gv_1s=ZvQ4t{E!2GfT=hrio6!0Y$fCsL zAe*S$FpH-1P1G&YU9|P_fXBAmS|7K|TmGg)GZk^6A6jA36~{I9SSxWdq}MdqvVy04 zJz*cAQQ&^t*hQ-YP;!LinVA_=I7++4kzJt0Ni!X22dKVG9Q!U|%-3Q1dV+p04+bO( z@6#Wy+2zv_qvor*C!;R(>>u_8X^d7ziOD7yTw(w{CV!H5PRS%Zb%EA>RXLXy{h<0G z+k~Q}FD8qIF&ZE7SH+R-a5NzJnT+Ooo9!l4L^HM#4h)4w5oVC}y)=Fd1BFT3Ny$~W=rwJ64*pxT$MSZ6&BQ$ zeVU>Lc#}z#A~!?_*WSVCXhmIBas6f&p=g^P5)?9nt1)dnV1yK=5J$Tm7Yzc{&DMqu zNe`nsZO9ImN91c8(=(-Vo}yT}Dh^Eo%9RVF))}!-V+iQCQx?fXyvH+b8NGv{I@o{$ znpX2*fG%6UgT=wG>EcD5GZDPq3_fsT(jDZ8PEnc`q-PvFNSY+eDoRnBpP{yN&lfXd zf%y(IS96QdBK|>UOI46uGOVqcpr8r}{WMU*YHG%xmRBI}C8q>=52Fq_Rq_v)x8-OB z1^MbroNB#Nozk3jm64jM8jI_N}ZrzcD8m5_@FI);ubdiHGs*T~tBG+R?SBcHCx3saGf_I?zPVmPC?f(?cF$UP(ZDQ?y9PX)y!NYC+833=5#T;nJU z)r`@6NxF`0op(MMAliYt+=n-0bp#*BWjGpXZYc{kG`~z(e|LFc3X+@=oX(eogfWdI ztg^JqHwSrod&6m_qMh4lIKMfh(oWmF0I&}_H(%NvMsCXwA+6`R7)|pl2zHqu%7b;`GlVgd*iB&D1;;$nHZHgiH1w%1fTkuRplM^8>FOCB zUFe4)jiuxGz>vn`AnMiEx>tm1*K{_gn!13bwZ=wXc5dGtc|TcZWi z9H@pE+nIcVMussaSJAot{n?RL)yZd@9!@JRvP-6kXEds!l>=8qy7_O>=%0(1QQ0+H zoi+!ARPy3=>;3a)5n}+(M<4=)30Kq#^Oar{!SrlO+GI$f;#Y-{OsYBU4-Cknh5meA zsnN#yMHEGQjY)g7kVPD&0y(y5azDqaDS4kSQS}HbUR({!1lyQ%25A=&^D?!sIF@hQ%MdHa{vW3EniSs^QB6eE4lrtH&kBE z?&oU%gc#3E1K9E=*c5;MqCn*<@KEi#@dW zopZ`jrAE0ORk@X7=@h)-^RG~9<9xs+p@Ikzx#kZeM%PQcR!$GxLNcYh+6p16km_B2 zeWGV-Khfjrclv2iRXOz`eo1C=j{L`JGjdSv|6}h@%bsb!k^?L9QWm@gAEM2G`vhRk z4Y*G)b*Wj%Jts-Vt4s8@m%xU;en{Zl>_rXVD%#B<*%o+;8}MLeYrP^|JCtDgi2 z{7k%HZCh_IVabHjq*{Yy3krGFZS4d`Iwk(DBo9QZ1!(O9YAv=V^16cb6;M0!|xdW;!=qzBvy~y|t(kN$XHM zhf2BurhB49+hB>X%byYN5Lh-uBG|aY^?O$~7@Q3>iTUG}fW3f6@3YD*t(0ujkPgULHza9yBvciIFYzY5<_hx`HQoFxEg;S&O#9 zmPN_ZQdf96IYRe7X`Rysa- zcO@L%ng|FVXEh()YXISHk_LcG7Hx+em^*1YfQvXK`^z@#CbCfUX^ybxE8Nq@CB0rz z*NR+7;53Yw5ooe{S$~V8ZFZpX3!Y%8NU|dB4T5*gAc7xkmV8+!h!Mp5=j|$@7li5M zyBF1H0RNZXzGGVk?5Ju{z+y5noNmV!t33UUK0ZCsSA2huLL9q|tM*q(F$JYh>{KUNm#T>aNLolbc!Wf}~lR*K_rcQP0JtavlQh zj?WZA+l$~q%)37oao^tk-4v$jCJf1I%r*)rWGX9#BX{5Awkgf0O5sGnQn>qX7KrMt z(YsGtZj{!B90Z;|779wLw>%Mr3Y5e~u_MAL8p9a#S@KkgIwz<^eJ$_E_dE&qKVeYG;6XdkkGx^8t+TXRy@6I@BFRwYOA#o9Vmp z9WCT8ieb0w;2NbO%9q!GLkYjK;XOUq;#6dl2QBYcN;aQ?K@H~cd*B-uUmROVU4KA5 z(a73DG$@T>=`GP9mn0=L7$jZ~b8nKm6tt!i^FNFTpo)B*y$i=BzF~2xxR2djXl91U zmgMNR2-v>=W4CRz*Mn1l8pWU%MIsFEV7Hv~(4`f@a+nJW=Z@Su=Ue3h11$-JA0xFa zGW~K;dBtx9@!J6aOPsGaAm0OO+)fzVA8g5wqZZ%|06#6I(|q^FFCpszKSFmgTQ$db zsy6C`3CqUzgo=D)cW6}Cc)~bA?c`hLlV-)4CybAp1)#00lyWuIg%-D(BnzjLjH0ma z8!~>Fk?nHU-k1POEh~p&d^zBjTgT6Bh!cAqCuB-tX1qK?gb_AAKFRrkEUJE(OCH9kJ5ya%yn#j2iU7` z@>-0psa*5-HN(C0a(^2~#L+nO4{nQ=gf}?n9mZZekaiQL;S+>tcZomVNd0!g=6CpXe>NH+qEP5V?BHtDoh;z9?P#}t-p*o9)Lpd!rsX2VcoVTc z?}BVjY)EqZ6Fen7lI?cn!kdH6&g{nO4Tiz{R+WBJ`NEu>HtxXti$7+>z|SH`K1dUy=b?xSCh6| zR?xvMGy)5^)LA*~QKdXK!R?IRF!>(ZKs>ACc@5sk#j0>nemh$IEE|o=EUjB8rU@GG zbzH(!>D9S^@KZ_oGr2$z@;Fk+IuswIiqdrSWHjAJ@&V9^eO;x&odu@aJ%E(I4KP)- zhsQvWZUM)WN5GeOi-i;;p23^W=a@FM%TPNmE{!kZ>x|GD4HiCKJO&0<4nn`}Avao~ zP6(`Us-q3c$?x1*@Fu@7FrJT$SpI8d6I2TLMypmzd@=r})dh-MRjPoluEaGq^-D8W zX?3MACDH5_=s=pT@LU64RWVayGZ`SPxsVB8l{T1%uW78n1Z~PrHPYx_!Fr|9>LH4z z(ZWHR#~2$q&C_VV4pmX*{vO?Qndv4FL6~j|*o%~vc=(bjtLm>iwU;pRYAxp>`|Qa{ zW0+OF%U9*(9W{Sa&g`8jylP~XXu$sGyK6jhwT_zV}#A{D?{JNd4c2m{lYBMQI!KHWs?XH-Kc+u8GCx;2Lz(e}OkiE&wahgY_W z=RZ`=L(XfZ{i3ZA;iz*LhH(pMAzZwF+Cru+;??8+DT=q-^){0NVw>Cxgm~aH_s(lz zaHkxWFwz|Nr6EvS5-pK5G!Gpcz%s(ESlY30=?y_HAg9td7R{YIpqKak0vk|)jd_Md zsk=oy-LXd~XoyYt;X79~ zXjXJ)lnSn@CJ2q6fD!+tyL48VU%E@a27Uml!i@4&vQMUqP)dI)^l?6x4T>9AVSC8>+{1R{R_FX+D zr62^DP4&Q{PcgT1_V=9vBSyoq5Kq@90fFQ!PYjA?crZ{ZKjxtzM8!(5P4N0OX96M4 znXCiHm>1S0^1_7}g2#A8lg6PLp847nuzwvWc&+9B>zqf>3ls*Y&R04Sw~TWq>X>th zU-)ihR-O#MIKt{ijvhd4cmv+uvLx9v7h5f7qXcHkl-_95nvEtFx>?orl?wGH$y$Wr zR$K*Fs7a-AF+$aSZqkeNI}D$LN|yeJFGuEHsp1$T-2V#R^=)fuNa$FSyp_OS!#f`|4zOj#YW`s*6MWb3NVN zf}A8}!ch3rQ4JvS;$2*-dP8EFOL7k%w|Kfy1revK{rFy0L-NUc{lSt<;T3|m`|rt) z-ho88Az8)azdkJo{n4l8MSqC@9b8}1|Mo_o{_3A!Tn?`W7rk@*q(?s)_OFN67nl8O z`UyVwSN~#gaoxYV7@pBb@YBDBXZ`-=;Oz38mh4^OC%vm)e>fQQ&j)hq^XM-|x2JJD zR;Ku{8T3gK-#_#&9!~C{!p1p0I1YDlJU>DF!u;Lto{|S9*vBAt5O0Fw;n82G2TAvd z!HaL~z^8rUMUf~V-A)+9jsuhQJ>e5O_>Qy#KXERmlt8z!*gygXO@1&FPX|H=%f*l3 zY`u+py}Km>wf`?xk3=U>K4*yFsKWK!Dv0tTF@2E*^|sLh8gZPE)i~7F6wc!OguCdY z!6Al^2>FXpSMN#&8Yyg-U3WbBm`nfzxFxCruj-)Z_c644_WjI&b>3?r^GiG@U3IPm z1P4TG5}<0s>SKbhOo%ZW!3sN4m9QqE%ZOZIx&fk{0#+9WR`@qy z^{xS|3Z!aaMOaa=B3uAg!gbM@(qzt^{V_pmU{X(bzbKde7GGnIQN`CTU#8oHOywIu z^Ln{nt#?%ZHu{|M^2ObL8&8)Yq5*8|uTS{P{1XEH3-VGkQOkmqt*Hum%yNP^QA+qZDhDP$IF{ z25WRzl{5Ms?uw}1!LM1sVV(sKiA+$aBHXmV2;tdZmU3}fneF_AFG~SoinSjGnXgH+ zxY_)0H+y{n>r=(mmPm*O5tfM6=3VFos0G;a>(w|1!B0ylq>j$cW@>k;(~fa=ZhRgG z?y%Hc_G*w-$RS-}?-7Dwujd}>Et-3NAh$Uv2Wv+M#kAHbO0Eb=Mfiz=i!tp8pHdrD zXQ?T?4**BSF8O;^?9#jZ^LZD~|FMoI+hmD)q?g#Xx__dfZc7wgZno>cZhL3{pr@JV z6a0k-uHlOBR26)**(3O`tHsBTWr>wd$s5#*@@q$^{DDNCK4Q3G-1^NAPu8a-NU$xzsL6i%~|K7`mi@Nm5J5`!qZPtos9*VQAj}TpDw!2 z5Qf?_kz8XithpymN=~sMTP>xOmeuSuNO09?W~|w_S@tIrNC5S zORt?sh`Cmjp+Rw$N;>gF5%)Bdu82|1oTnqACK(0lvH7JB_f#scO@Vq}9N>$K(r=`@NzM+)~sL1?^TsiKF)Km z?uY5Qq01ul#{I$F9>&kSV1h~fD_hB7lHVo>uIfJ~28{0s>{EpoS~<>CpOOF=BUtY# zO2k?lcJuD1Um#9x1{tj$Cp{Ibk4VjmbpI|kQU@cjjOG0e}i)*^Ed`-C} zD#uF=O#DZ|aXJQ0dU%%G3Io{t9j zdx};y0b;#)|Ppi>EgazX2c%m6izWyn%>k@c#<`-)Z!*}xlJEHyVInIS3Wc=4? z7lv#XggJCu_^Bl@mq;!?QA1tR+Crc%(#ms$;FhBRVirPF<>v=AtjkU()OSucuOvB=li5+<@|gI)=4)C;UNYNFVsA| z((~|2>Up9U3z~`Xh<68!gDA?i1pg>k+yAwGqpw}X=tPcYn`hN})pXQw`O)+qk6rBX z(R=^lYPyUKp?W`*GP?g0l5e{jjQqYg{`Y6^rV>C=I?p0iZ|BqPcKQ z;BRIi_6zwGEDqDznfu2(GkH+OTLbD#jT$Q26yZct#NVge0JnskDCvN=^< zAW)BLK|KqdUr$KsHxAIxxat_s?GTtzvZu@>l@e0RN8E+v|5AKykzvdO%1(IYFjUEyI5{1k$%v(?bLWiHpNDHS@tdsGiOu`Ax!;sn;_KG$IhrS1T|6@Z!IxKIYiXCjo zWeaI!6~pOJE4-$N;dkp))gAIusS9Wdj1Hx)4|)gdq}ur!Z}Ae8)?8qYtr2j)#8ps) zm$gXUef$N?3|kQ?)X8aaj_mHfn@%V0nWmSJYrE4G8>v^=L>=^27wFs9QDO>0_0QJCC-uVj%OO%6ZCG8QC z$KjxSkh#m)AdD)9j{dEP_tF&OdU5<)LBm^06H+sHFn&0Yx^z5wFox31iWEQd)Olr* z;=!&j!bM#xRA|NeRgnj~>OoQl4m=CzKBx@yU{^jUb~~*5e479$6q?#NJ2^iv^7MOB zYFYmdX;Fpq}T_mYbgn$ym37!-q*<8`d9Q~JkQsX1wgmwjbCp$s_VJt z1cjjdiu6v0X(h{2d6{qjcB66BBtTXv0{A)$M#0Nu|C0*V^fO@Qw=`RV;+m@cUkozt zXHx>!c)i3}!19r(@r~N$C`AKZyRu!5KzXa)j$`slX574)lv~kcgK0D3`s+CO={ojW zFxJYh_4e)TU+Q+hH12FEPo^nFgp70BTO&(h59SLklGV!^k63QAdn*ees92uc$tpj2 zbK6N$ocg%lsU+qt)i;~Y0`zgQ4nfn5iVT&x!OJ>9Ml#+%Z&%UxEzKF!N#15RQLhw1 zTVSX<@1W@Lg4CBlmC@*yvKK@*OMW6tdOwDrZh*{@m$pu!Tw5LH21lc3-->U#QFp*@ zQb4R;ua!FEqN}aI3d8A#9AJ_v0h0eRA^ZAKdLh88FEWP*3i6E~6I?5EUI+HEEHGIV$Bhor&k9z#IeRf6Aa#$(8> zS!C{z6CQm>5jW?!7J|PsB)ecMGGJBH-Wt%DxT2$+?hvybec%?_zw3JUA)@3?n|)qNeDrci1J!Jqj*>BA;oc`Yt@l!=Z(Bqn_xE@DH6(uHpmGKsEI zczXa0|Ms}E9K)}jg%X)vCpoG$E#-RqThY5Bi4V5X-o)ZtpRnN!$8az^0-wPX9K&}> zkttMR;ONr^+l258Q@%d#aPvlk*^TZiR7`I4e5rhHqx(x;PHimPp`)3N?k&W`M)yZj zhW21I$Df@|K2*~h-A6j*S&hdyaXw9q21)drsxfEb^NzR~VB0>q)}@JQZ7s81FJE!( z5eLGxy<;f}b=i6C7>>e(B0@BrWmw&2+Y26s9=U{}nk8@CG3;iK| z_T!NEP|g3?uY+lfZL#;7g5^N$bjT~E&RuC!Hj6v?%xO3V_D57f%^4O(!p&@|{Fb!QooGMGK(T z)WuUqPNy>$+g|h_{$Eu|LhEMpwzoc%=f`8W4M(oDy_zrpZKUac(-zG`H;1ycUk%XB zXDxP0Xg08&g)}b_T&efi48CFWJE*~HGHLOYuf$VUZ3eS+ddF8I+is>T{u;3Gm!bw# z?j6XIJCK5O6v`c7!s1DjMVIpx+g|iA#$OkYTpx(64{98BSbg~xuU{zlY&%)8Ep;DA zh6&&L_cro*vgueO)!%FaK+=tKGGCSvC=S(V%MC*Gb!@H&*tT!>YK+GJXSSF0eg=(2 z@05sN&3QTaEXB59I*{fQlXJS8qh25^;C^vjnV zJ5>Z7OAulC<*(HJ%*3{!IPr;zU0ko-yu`LA5LTHjLbgpU-AzjjIc9gW65YE64jgoU z85P*9ioflg#PB^jKeVG&xT6_~?n||skmy;{A)Y6|33~;hvX{Q?tPGF8)p|Oj`^Gu) zOGg!UHyhEnFFC8}c88vf=sSeqJv$lX1m+^vIB%B@SgR2S7JLqWD4Xdq_B7&_j6dTu z<9y$4JP*+gXn)IPD+tNCAmr_yz?3U)Yzw>X@Kdg8%|Wb-si3A^A3c>e-3Cx|NfexY zJ7fZ)``WJghu8Ou^?kHhPvnxHLS-pRg`JK*H1!*gl$w}KeyE9w}R1pKFaB7Ccf8?JQXf^uNc~#27wHf>ASoD-e zemail;O^0(4I$?-1n9jI@W3eN1s{#ad^)ekyeq*y%H`^O#HaHJ0>Z9-k4ED?om=tS zGBtF-(Vos@h#Pa&laB9n0##g0YkAJO zhc)@iw0vWjNbjCY-N31zfp7|$URW8WbnFGs$34-oAGN~2*y2q9h_-){Zf*ol^l1q}#w;Ld4QAx*ztMgNVrYr~a74YfO|BibK27&P$sZH9u@ z-6DjgnA&zM?_nMHFwk&!1**rzI{2|>9V$-}EaNzubB?GtKs1PYz1h=r8|}B#WWS^B zKZ7yZ?d^dEJa~f6Jar-!*l``gHJ9|*hv;z5spE=rlxNeVqXOVZ$MSr>spHqF9CH26 ziWSH2h7>Ry-4;SjK17D)4ILY~L{!S--gs#{Db?v=9U`xr1g8U7(8a}6B)Ky>IAL`q z3!^dyb*)!it;2`k92@3>QwK_FIydIBD$6i-me6$W$y@}_Rl9rU&3ORVrH}dicVj-) zKHL$KnBib6PhdLMDT(=3#on7HE}f@f4pfJxUMi$dAWrAI;eq3HbrO}eFcXxdbgdYg zpmcrLt87EESquZg*({XS{_WG3cYl0}qR&z9>fPflvV-v!hN;d|d46wAn%?mKG1@)f zBMFF^54&&1Y-HovUtXXvwZ-Dzs@G%Rhe4~XGv@-%U*V)*ph}p!&3!jL>N$=|xzBhT zAvwC;CrR0{&edl|`a``7_FZ*bM;?|!x5BhU;&RA^zp}#b{)p&2A;`xgwyoSmoA8`qoX-BWZwoqfw9f7B{B;2=%2jglNLh*-XCfHZzh zt=OIGrkIlJq-1xYNr4vBH*--=dgm{9XZnzGHU0Y(?N>nEhVYKZwfF02vtBO>m(Sl&XrWefVAF+6d{4P(3j|#i@tJ9I7QR@TLz4DLelWbKFmjI(GTrOv zqLhx+M@H&1id*ULV1e&&AxpnFq)!yTx7)8brJ`X-pD33e@9i`&O|R$6=Y9M`MCC-1 z{A#vd%~555h>0q5xL?HH51;?%Iew1EpIs!O$Z8(r6J)MSP+M!%yKrytyjpJoxp*G! z;{BLVHGG=xo`@3g-&a_7x`_5`^LJ+S6B@niTGV5Si@mDN$g@)+&ngTE13V`C#jkrV z;Q2n>8hGA?HPpRWVZhRe@Pg~S_Da4p^@fOuZx&Eo_s0xL0!# zf)N6>uSPeRt}uNcl|Iob{+<^UHipyYqCV)ci+bSZx~b;(knj5T4YJC7Cn~Yb*qMBY zYjq^gq9rG4D{#EW#bghXTd>z4NbAcnDPGzwCiOt?jJ2Wv5n;S#?J`QS@N0%-^NfQa(y^I>A(T(}0fK~r z`|RGYz6wYf<+wl+4&=HREYp*R-IZNg&^M!)+L!Snwbi1&lLiXLdMrfl)_Vbq0U@Cqty}{PKoo0>MO>GXvfuUD+%I#4Vgk7Q|@0sJGd(@rr^4 zn;r9fwuq-&W}j*moAZ7v5YR)hQfltb@c{Xr6)Op*sw}Z!J3+*G(gLdJ0q%&d-Kuxi ztJ!ojohAF9E$=+sCI9u^G7zCOu9*xcO?=O@eKf`M?W;LES=vWe?~(!`yQgik;xVMU zC2ve(6*FR``87y0_}$TTKHtW>U3M=r`Lb0fUn3dRVjRvtSlm3Hj`MucLq-H#0SiQ| zP0d|uo-!26Lm#o<%|fcne2-ldO!o3&^OXy@w#vb)AX|APIn_Xv)!bLgsnd3g5wm6V zf;ac(@u;pz%Jj=?I2K-o-V!oEl5!7K32VmwIhDXX#&Smzy&%t<*>tz}E|vE8*h(D8 z4=EI!4yvneVDXZc4V+Vn9YYZbZbf|G;-BXI$oZ-aY>r9l@-Pj@IT1Qc2@V<8T%BvacujEsl zwO7({3S&4nPRoVIDOzOa&hO*LWaW_-ZE~9f85`h2s8arVvs@lYK+&WLymIaz*G5`} zQq$bOy$a>WFK$UFakGrCJ1re;{OQ`i%*Ne5VE4C z+I26Cb&9bRYN}2>9w6Vi4u;osj1&bL4Gul$Adpl`PEE5#HyaO!)kYqozs^{my?F~! zP^P}yQ2+VEcD-U=pmeBP9Qb;(-bC)jI{?Rdyxcefuf{J{QTS%WAma6R=aCF2wxL}S zVrMHiRY7iwXu6t@bTvfd|1cJz%DOH^53@c!LZ&EU5Uv&Vplw)yOj8vQF>&*}(j}qE z9h4SK;oJ&$8BnEK?-Ip}@#7X1W+;Y3X(~qj+*7%NnAcMcEsq{1h;bcUWs!SuaWuQz zy?uJl(+d8p+U5-JPGbs(|**)ns57O>HA^WA>y z$$WLkzbZgqwvnHY&m!GLH!z-l zeHXnmgs;9ukI&QXoDv4QG8y$JY|7f`R*r0K8-y5A_X4v&(gp(~I6O!4#vTH5>b*W} z%*Av@C5f|dvp=@O=r|NkfXs|GAZDtPL|2` zPU5If1*u$ikJktVzx&Q@i5yMUPEhh_8wGH<;mDK$BtQx@H^?x7p+3k8V-+-~o2}mW z#`|%CAvKTC{$vn@|`J-j0M$2|MP#PZ&PjzQH7tyH20Z8Cwix!z1Eys$AJ+E4zRykMhI(Ni>^*lgE7 zSe=+sovjh5M;A6gAb^4MzSB3iwmr{>0yD3PDd&^vR@<}wfmc3e%_p*76QW){k!R34 zQtBX!@-W#6iaw?ot3)4UiDL1H#+0POj-)oaROgt)yc^$qTAqzQEh)rql!SE2-vX!v=2N^`@EjjhsJ4C>(>a>cjaI9l?h%?0YzLLD)w%Q(R3Jgo zui3j-pXMB!*$~DE-w+KmzS>n>rP~6l) z3^%t@Z;6KoT3f&mSjGtb{|jQ488}w7j)o*_5n1>ks2cej1C#hiJVUR2hHC|w9Et-; zsNs!B7{Z66TUlEA8mU?CUzI+f+mbgVbs=dMy&(%4QpU|%N_l%E2u~zzKNR-tz1VWAX@Drqgij7t( ze(m}89?2?>wn1G_0{^22fxZ50xp`=pc=B<^AbtV|PkzAmF=SvvS0{?~4YyUefKOxoQpl8z+A6Iw zY_OgS|E5=(*BKTD=GZfiM)$f72rTq{Yq(7rEVi!DB1e26pSVny^H=tTecxNe$4R!q zOs9)vwTrhLk+&aZ50*z)`-5YzZ+Vmw6Gy)(Gd)rMN~NkLwG%z3pSDl*z(LIPN{Z{0 z`HBZGq%8;6RM{1J9@8N#4XjDot89QBnLxBBrk+s<_limkjh}K^@Kfj~y3~-?fykBX zsJVA$83Z1XI?cXWbgK?Tp*b?gAYEDbm)-t?c=!`+8F zY#|=Pk6(V+H_lW%XG9l;H^`al01clW!U;yMeJscleO@-a$@4Qbi1Rb)5!`Iq7QlI_ zB1P{h9?u6oQVA=}>)mgYmjq1<3DS|EaRBHw;OZpYvZKGEHuD{e!Hviaf;v5<3PZJt zPUY#&Z>xr&we#Z@(dRwjX{uWvPJPBSLGUoPL4dmT9o8Fj7Ra#m-QGlvO)TWIGsaJ^ z-B(#RS%8Z0@0bxgN9tP$)cPB%+<;i4L9A_h$70yIbX_wVC|fZ-VEzqxDVV8TaWu*; zB%enGAJZkIf+5UD*6?`&(h-%BD`xEQ$fQZL(`tv@< z({alDS6&|;4w`OR8SO~C=2d@_vq&FXuS-QXvu|vWwSjYEC;>Aml)gW$(PW}_V8R*i zzn`ld;dF;uI`d(Aa7z^wE5D}c?!>!pplKy#wE@B4d8N8_iIjAe75!xVC%qw9QmaOf zUzUD#!EVYmBgNQJ8BGBJJP&7=fGS=1zR|lZeA1gtaeOv6Pt5TN5lpdXy^ep(;*IKK z_C{xl2m4^3mNv@G@rhiP5@1(B#a}lQt*QnoS*?4NfPZ!2lFC11uyw}t_M>QxCtFs1 z_r%Hx1v#R%ip3;|t$uO*_6F*LmX!;9Z-mWBhKa@GTfD$&>;o$we*@(8&D&kG^|OEO z92ByieKAc|ikumejjp#KLtJMQCt_V9z z=z?qJYra=TH)FI%j7H1s)gfzpfXtT9Tm0!q>up4jM5#kAKP@kYxr0qUzHslAL`%t6 zdtf7_7ZzyLI3s7gcafcEg!9x{%-XlK7?KZIZNQz67OM>gwA#hce>U>y7VGK!J6>GQ zJ~dvB&}jykFq^gtB%p*8Bmx>DEKspZAF%iyPrqeL4f@gdWdDSo(|FDub%II_AOm4r z^?Yx3${{jyYD>sOJE{`k5}}-E0x~-YdYO+`@LkRGnnbG&MKQbWOX-W9Rz5aqQ=Up24%ouc$Fd^OAKv_ z!15Y$lwdwWwME1YVWnvEnV?+JpE9^if0%@W8Uvt)0NC+U#x(8Co~B#-)YfctIa@Dt z;a7-V)U0s9d%2iCujcVK6RcwzO>O$Dn< zdko${47)JnvUl;Y-hRi>UR37kxhqU)>Z3N*SjrH!mr?T{aRDhh%XioJEv#Tlh^*QR zfWpD#p0|;^YcU(5jDH8#f=!Zfi;93M_qmj%XmFDyA`efAZX%)2^c<}Cln-;Gc z#fBP^FY=1KK)HZ9lzu9gSQn63@3h2n=rOBad5EDP8DIV2a;TZ)veQ2K-F^ebOjlIaQrRoBm1WJHvwty?c4eL` z;%q@vowzC}F~WEl>j>6JgXgs64?lZ0PxYOutfj^ zT9&_%XaM#KQH@kYP~as?!|z%*OLGBZs|QAi5mM=ZH2-dKMQ)s)wlpZu#b~x=N%YPM zDBpzc(gR!voP@3-Z%;n!ItD#z(p)0aAhHA$5i;#1MIEQ;Qp6k%CHIuHqs3vKnSs)f z#i+P5ZkZ8%N5yp!j>3?q%&>tRrckQajH6ZjJ@Y7!2fdqRu`>tnA*d1DjM-1<#E7{5 zajMo6_w`AVza{I5o&Ctx6Qvg@3A&AUyLgU=){-s$XFf;46EQ-n`jowIpRZLRO0K** zm*D+qY!#v|nWwpN^nQdK|Mu2SpKFphhK%Nxxi3#Sp;uUBB<(0iV`N|BXs;TW6HPS} z`8vWe)m)WDd|A^XR_{oDzPm|tgD);ixPzJpPLJc8TS`h&Hj-pbbrKV*QCsfs^*cZ6 zfdZx~4|P8EW>M8S`8yQmp!g9#sj-wC4^r6oi^wB!K2Xuc%^16-kbnJ~$)Kx#93{Al z77%4zr&I=rsrjQ2W;QiNxHOj1H$K`NNvSS*Phpe(1~Ee_PgNNlW#6EUT=-wVK{gKx z73JL}Fo0TH#Xt6G_KI<fS(=CS35+0Qj56(>WXh{s_`|f}o~} zlgZ}Qs_@{eR?@m6Keq_~aq06$$gQj#;*DHEbjlDxG1ILw2KTCT}#p;OQM0G(l#CAC_9r%f{3k?FI!ij!gzW)}WjdxNI>Xn4h=PN4hH?ztrz8vJmFY zlLp*=@)8Gx4QV>jW+A6IW;2uMxWr>2n39(w5C^!i!L7>4n#$hRhqaC^^Rj-P$ z7mi$YsQze&_X|{K+TmS;w8IJ91qq3FQlY_>G~#i#_-`f>{Vo*I-Z7$`6&1g%e(nQz z{oYU+SKen(1rf}|d#z+bu`noSh`tjNvr77RPB6vu73Y*vGDcK8yvU*j24;#TsZ7n9 zT3a&~I9_7Y91VhNVq0_~8&FJ}7n&j2$^))WO2JxnNza_zGDoZ$;=)M737{ zGyAo3DUi1|JhV!=I$^9)&!r^*o#3@=sF_()UPGPr zSJAO$Yc~yG3F!-$58f#C8U2CjE?Vx&a=MQ8MXkn%TJ<->8x#^*j4fo3I?|M67zi zoeC!hLglAvw8P(%1ON5z*4a-Yj=z2S z^6rmMQS=!jnQr6h{L6+|mRTlh$c`Ggv80FWX&`Cl1mxd4Xs~i0nMzcMA^C;3SC&-N zj|o4|OEpwB+XxAxz9CxFcu2B^yPZAe?0u`5SBi`(LC3v|mC}Qb?MS;d;JlSnz)#p3dv}zQKoOC9${WWh&sV?6O)20cQ=G?W=2$M2^7`1Z<|6V^o6q%+6rKfpc<8Gnzl)c*H;JS z^=0N@;mdkJ>4gs@U#Xa?9RoM87`bxR^vo0w1(t|rPU>z}!N~;?(cvo$(}b%7v<<7<*&bh#gUD>a{$P!wE{b=NCLIn!46i|T8+ScJ1jWrdj=VPFhL+J6lAXi!marg*QjW^bm!34l%elXtU-l~-M} zkEMq$!S(cGLJ3tekq0FPU-sSczOFhy@ zxdQRGkWqA6NP>bX&7-X~)+riUKFdIEAO!}C<#@QGL|)T7f-X62-84NFtfs3{r(2`u z(&WaJH$t48)#%WCN3v$~6*UN|UK+LYB15WGhGhdo)}>ExF(#UG5+mK(JGj>8Ygg*) z+BTWD77u{T1+&2fs+Iz5@`G$ArG1cptv0U_U zD3vChFQ51Ek1|$}rUpT<>l8HBB-(#o;Sra3i#ZiEK%HWw@s@^QBR!5!=p2a33*c{A zTwVR-(|r#NKW9iaU%1+=e75S-t~kI71i(iKnSdGM!Xny@u- zfX;!Vvvav_ssb$?@hltf^Wg&lc~$t*uqq(K_XQ9n@vVO{j8xH&u0MeAP=y2~0{z#g z;#_HHE@yHG(!`pt%C{eK{^PHYW0EfO*v*30x+oDK?6fdpSps3`)Np$aK~DN` zL{t!O{tod=i0wZ~Y(r`BC!Iu2ms^W(?^;~3|2o$y*uNiEoD#vkVkk#%^zY9QVyqzQ z>fIxiM8(_QFkbDRw=v$+n{M~<94`?4bMZ2QW=BzFjhv#b3{|BpMA9HUJoAk`=%v2r znR;TQH=M^0)91zBj4>VS3at941HpcV#Xp9VO|%4_QFjB>#|99ysE_kq|E) z0Hcb?cu2MJO)?FQ;+CHj1V+BfJgq# zs0p9sfJdgx$sgGV<+NtFIUH5_wVsfdH+i_8^YE7gF{->fSO5h|MYX6yV3-#uZ4T8T zFp3-GxmvtpnHII)MHgCHlZU`!m8f_J-N6-nzg}rM{&hQD?eNS-vR*l^ZH^YWE`_+a zF%mLJ}xc-+^mxo%FXfRTX@_&I_ue{G6zsPzJP<1YBK1=Fwk5H zbe+SY;7@*dMq&7`WkXkJdqNW}#51e4mPF-<7<@JWjuCEKCHn-`^#4U*%U}zyIpal- z^=!IlVWu*4QMEl%fw!vMnnGfiJh+>G)|XsAdrgJt?q2vMm0oz^cbJ~1o-iODn66gp zU1K?=SqM{#F{oaL_{S_(eF)LxGhU&Sl#aJ||5zxPuD2gzi2e1?+v(~tzKvhvEs87w z6AzWDI6R1&SfY>IXl|SA?>qGbGTdk;I9qKCq{=`sz`UIfXV5ZlWshnXEbLyAEF%xWH71O6J(e1}2-kT;WHOzu%VmY?T(c8)W zyg`X$09TL`g!GZ`F6tn1+qWScVifK#ll@X;6c0_f8KMVd{p&8MvliMD$IO~Y`ppxFVaBkvp;x^+(cvYzE{|Kh zGNsz2rmOw$VCm=QlqoK!|elq^^1+ThJ?--E3KnL-2*ZHoDd{=yv zr0p)sHJ45VM$RamudQM-bZt^!5b3Ri^Z6W^ahSz2#D4zn@n(IWkEcKxj7Ha+=iL*I zG5a?9VhP^xXYI8JxU>?2B-1VPfrJ%Ue^r7MT9?(zmWa)gO;1V#8B-r6S)SS7f)x#< zS3ib>ZHF9K*`AjswKfSLqk=%GwSl}v;S#2Jv=}b|7jkpdhjUkigbZ6jQEebJuormP zt|9XjEy2pl&|q#5xJN#CsgSCJm(tk)gJ;1G<-sAHR(x5;2Nz+{6#&Ac-hOOS+#ui# z$`Azm@hxqIKk!Y;113uCHx6qyD~V{a=T1)G=NYX)qL$AN0<_R%tLH`lZcQvzb#S(! z<6fRwib6|sOCjLqxn9S?H}a1P!MBUZreqB#9|X_dphl>4b<%E{L@;1ou3wljFvgf; zr+iTH9k^v^)5Gm$cUCL>BRZ7oU}LDNaQJ`@#@$sCdnS~IL&&%L@*nnGLdctub$AR8 ztMrL)O`j1ljG%xK6-&c^?)E>(xf}{e`dvov7nlK)%>GzBPS%TL_H*RL%sJshk-e_j zB`lHk2qzVhxLoKIxSFjuKh>THl&3>u@=i6v`WgW?VeL2(V(%WHU~Bx*H^+5y=ekzs z`a@XqIF~sM(uR-06v-D?u?&!gi4|KJ9CQ;NcPR|7F(wz*&)sP96^0>&Dj*K%j~J*# zBfl7cnSJSl8X+F+z`hizDPN!>T|cY0*YOMNn|`(!s-Rd3jwQn8*D9b-)tdolA<89} zBrOAHC&Qb;C>dVO;&B#Kp$&&L+YDdA;9d3$f{*^q2u{pp^d$!-V164bI;@3xAo@H4 z5V$)Ch?}M4h^8{BRtfac2wQ=b|P?$e&H$>kPlqJxMzkUk>e%M#x>0QkeR5U`EWk+>>+rw`p zW)*`G9ZcbEMf>eE*{if=?|hz6B?^g3{m434Jjdm2Y$s2(4QC5zXgDXrvps;F1HAgp z9IJ;zZ8U#)ICKR^yxrJ%2Y~irtaKl0nT#r;mIXh2)LI(v0&h%Qjn9XG8o!7PtHSR; zI!A(o`xh`j=IdoNoz0MANSe+Ng-_%YLySIrT_&@|eE+bCR2~P(J^IU; z`I~7U%Mmr%QPlec8wv`{B*E&kP-)sIx1sL}GWjbdcN;>p{=JLR*t%}}$vL#AmUQ#1 z>nTCA3YI34NZnOlMdfs%8$9jqUZ<+gEuetPxo~IYS+E-QS|^UR5gIjjK7b>vO*OB@+(3E?8O(&0yz`?v>pWo-DQ8&}4^S}QAD7s!e%1S@cjDV~Qk`7ZZl3KR1_ zJH=Sllw8yWhP8`juY}bbu%#<4R7E%5r4krk<(!ZxsRhx?vM$SLL&@ZUlwm@OAuT&=I4mC zlA=|$NFIWBrgv)ACDmEFc8yY<#j7fc1SVTSyLu;Om4(zUF=^@%BW{1&6o&i!ZBJ#9 zgPUB2Hgqy~sMGEq`Ps6E4nzQnuhs0q3GZ@Ow(d(;dU-$BpbSx*5+UatG<>JS1w%*3))9rTplY+erW4iJ~-1ZBcQ>Jy&h2%9XwdIymReu5o zt`Nzhd6ajlLHdnmV3OKa8z9A_&`;YiusahX1e+do!NvI&*^AnLAJSA z+n=9g)7%7h51kdoc%d9h%WHTgJ!-@4vRp*HbAjh>gZ=R&El#v|UAN;vOS zuF*~~>&S!P@til2Gcv1O0k1__`9&^px&4(pk8&bJ=VH~$g63*ABMHjOeK%#N5N{)2 zeU99uY6+msYQD#y*1|-GSxwD)rQ-KO{-USEY*y%IsJ9a=LE9ujS)pjRz+92dbpMp? zK!q#Wb*%8!Cj9tw5TGCi>Q9Zyuj~x7IodDk0RU!3%?UEpeN71~#`J|aSOF}~!{?oex9o{tUbA|9`Zfo$+7uz41{}qLI zgu&4}+dZLeaZazZ6}bQU4+_m-0?ACa6y<|D;frt$l8eAJWS6}1%NJD@fa_%4n&JJh zMQaE4!RI2-WI6$O{T;L{8hhXcx3%5kUzy^{S>NG{hiGHHps@FbQWw{9kBAeN6^Xi9srOv=#o`QA1f>!xm8|&&-{wej-_C2wHXu5LAKp7= z>0Yz#11xEy3o%cGR{AAFv6HAi31xq(L`4AR8PVzL8&J@hUT12H6U(Pp3Oo8vN1h$B*c7vA##!CUPqg4GV=QVu#-Ep6{b^ zbpMFZ5zW2-|7M#F`6>Og4b=qiXHOAwGIp3E@(elGRVCu~=tS)mt7V4x;S6r9?Gb~! z`WXs`$xHlXG)nI2rL8jWiE7+*PyF7Ea?-ctQ{@A9jP2YjsceHvcysy3KEW$kQUrx;ZzO3GxQIV5vvP2H*CBDgSX?NBcmPXYlNc!HJ zs0Ja6d-l}+LQaNkw&kD5PI8?dPkR03S1}!v>iGfMje~~oQwk6ND;TMe${rC|By*sE z1ZFxJ#k^-AoE{#iuCVSI|@7eFAJc)9>H9bv%N2MMeObM-*EQn-VawaV?OrZ$aUIB~* zkk{eaq2<{-ee;wfTnwAsDhNbHx#SW{oLZGsf{;Uj0zXSHF%<2*NKq9Ll3^$re)Rf467MR zh0R>S<)E3(L_V7B;1wnq+364_WY+0_RMi9QZngeuJ&(sYA-z3B5F6c%c?Hk`B}hpz zK?z2)A$%Of9~X)qjBj{S0CkM!%muWpN^l@e1vvEUeC)a&<)jshZJ8}+6l(+6BHk|Yx`_Uw@oW@Fp|F5$=`HIxAVr^J=)jIU(ZsV&nOiA$2dn-)W7RMj z=+M+>R+K(%w(AFsBc;-rd&hFPfPJ{fZ#cRAq~4|6WVQaT==M|ZLhriU!=(R-I&A&( z{O3_I;iB`o6!K)uRT^%N?_6SZalCzrjpvzke!nI*Q(75mE}Ux z1|FDY*gyt10}FTzd(Sl}W3rn4<=73n5x@nT*MTPuuCEaR3;?BR1yl>Gm|sGS zqSSU9bp}Nu+0Iq;pw`FnvU3npX{ZzpoKkbPMI!<#w~P>y=Lp`h6BM$RoAKPzmMHnW zXh00^l?p;2Cvln!g4{xGVPInoaC@Hd@(Ojrqv;E3WKsEMCcG~3o!%&^{hj`3UiqDK zn}&(%#RDNvnA;1SnGL%II(NA%vY7j(e5IaeE(2ff{nU=%c2R3Iw zy>vt_FgLzhrc)5kY#PPt}7o50V+8T_Kw4rm5bXtl{jauft4(BsEsDY2;R1u*ImD)d7#KJdhEKD|*{nc9hrbInmHf4i_571A5V76RNbR6q1V zAfySq6DV*R5PZ-v@?Fptg%n3ZdU0sX7&XOOOj9Y1=!vFuu_?i@21q6~NbxqK2%&%s zX^bHBu%NFb#F)ZeBc%upK?4CLC>_F}v$~ZI&r}%J)5x|G2v;qKob8Y}@5&@Cvp71S zRdsaK;-EQw`WSN~5f$uW@<~^lQ8b1Wwqn1G_XL$zzN%BIqM(}GwYMlKZG{(vA#$z) zS|dD9Yj#9cDQdj7MOS!93o^B((AlzQ0<&E$OQR1^4D;W3i`=_v&p607BrfGP(f4@z z?I3%lsOFO7P+XX97Q&89N0l?4o9*!SW-GRXxs|eAp|hv-K1kh6ZYiAjRw%iFqhH0O7yKwG4hj-CBqN4}W=b|M zE+hi|9H6yU5oifNjyoCuxtd1F4NR-E=ZGEoA1AN!VhNAEvw2)}Cm~7^`Favj))J^x zl}VR;l^XWw#ZditI-cHhyu11&5Sgi`A$e*X-%OtF_PyS8fu;qjh>P|BD;h6F z>WUsN4Oz+Y)e^d*l)z>~!=HFyr{oJ2)%YAEWvgJ{w2{M*vb%Q0p-0G|tv!0-Dc2K0 zl&c3V2+3IJ)^L4~g1w&8R|@L-jd@SRMY5J0q#)jxbR9Cama32BY5HX8R0z77FTD_a zozBA3O*V8uNrys7ehJ-p`coZNOygT~g`K~(5JQG|97VNHh|&stEm&ZxpiR>?Zf=?ln> zKL2y!fm37-61}5pjp+U-dMTsz{a+Y33^NpO-!T6w00MP+3VLW?*56|8+=|b-_QIDjR#XQ) zG>o3&#RiXQs)Cm&zKRC@EmqX-^!jJy{`J3IXY=d_QzJjj3HFqoLZ9_I(N8dw-h*oy z#7oC&YY>%A5cDp{6-XBglKr$(!HCR#ht{u4FMXii7LB&Kp72ugwE&GUzyX$37Z{z? z2nQmB_T_F^o4NY0CMX--ms@TJI{AAe;{Q}49@UPS%mcK@x(tajDgMAIX8k2Rf-m!F4$bFuvO-BFMO9XaEr@W!)K}kmS%a4sP#C zPpL~V{|1`+^W_c_4*v*|dPl!l#tz$Y=ts>feNE8c^WJtbx$x#Phw>P4NC@-+aX3G?<2 zw}zJmm55?U`uQhZ-w4DU5bV2ile^EVa1zIw$CQ0QUPYx?RDCAWQ~C`Ln#YO{Q0{IU z?eJK}6utcutWFJ)BPuj{T(6P7r<4i#M3kfv-B7;^jAc8i43i#Gpb4itNdA6?030jv zWh9&6kuUho7tj;4Fn$*lqJvxYdoZ6$STw(hwD87e--ur0_MN74x0((rASr5vZtanl z+RbaqRuxp7_Prd5C(>zI((03tPgU&(2el}L0Jf%Og`t9}QlzrUv^0iFc@n9ZoI5oB z+Sths>%&JEVqF3S9puuDiP9J$F z*55l4Ca1`0u-`B-XB8X1cx$Idfhs2U8VP`5hq6%c#A(IW^m=8|DH2TY$`m{!G-b=T z-qZz<8UKRdoeHwN5puuSPq%0(rIhjhTGwK)9@cN^2ECrXf(}y3&`upyUak!)?0^qe zHW!VcQ`Y`ITc}oI<`9KBs*+~J!vTHt3efy^v^wqRL9uDF#sAFbm>hkew3kgqmu4Q- zuqK>V(cSBVjY?(S4%l+#fhVq0ORU=y=J67-cFL#rlg0_3`k#ASQE2Pi-iRAW2e%M+ z7%nFaavBknj>MQEkKHWI({1=)IOYw5#|R*n#N#N`{1GmRHx2x;aT@qDmhC7F{Dz_G zO#Hqr)<7?vwv0V2!4iM9443Xp_^3F6OG@jU!jyqKg>=dXP3oSE--+7|A`uTi%Mo(S zsxo|_=nO+gl}+t(?{8Rqc(d&lRTppg$~+XF%94zE(LVP5MI;%!DMgGN zk}Qk3oUqXtAd%n3J-qm+r7jK>zyX2iP{ivWMzzF|x4kn}f9N~4DVFlw zXo7|xew(U6IfuR?r~b5oqIQ+rv1a+a1!#0OZs;vs7SX5tFeR58- zL43h(bEwM%WioW>JJf>g7f)}(&iR@!ztKSd@O8GBEq2?%Za=4ai?an<-qaHuyN%=L zYIp`x?0+X*nFnxd5l(Yg`=h@do(6J!;Qmg9b6Zi%@^(5Po4UHc3;-OC?`KX3A>&-q z#BU}cDGE)m8M2p^v=@d z*O`t-bG~>(mq8c~ZnNDfE|YGWRSL6BCl~}Iu3j;Zxbab3fH>M{<1SC#W!Rz_R-K@9 zgcq{69LwKulrtiPR5=S?gXR&N(TwzV35K{PJ4R>!4tV z*(S?mpWvl?F-obN$+XV?;(E6*oz#d&tktWyWAhF>#1@0ImgHV|vQOfBav?v5mw(Gk zhw!9VAi(`Lc}Xz%B*DACaIochi>HwA67*CH240NSTBZ2PgiLlzb=-1k5ozHfw&l8l z98wFyRNvKGo4?4Ae;z>KD#T#R(O=2q6+@U`cCLGqb61V9Ev0aeQ8)pw)wGK87d~R9 zM{eDveIi$e8J*8z43^W)I02^*smo-%z<*^#2lG2g^w4Ibv@km>HOT2p3v{TA)klUA z>VfcukBLdH3DGLa56TJK8h4u-U8dVu9U^m^sXfbkF1*FR7aP}HCh&Iw1R5Z~58MmI@^hGYvdu0<<_ zUUb?yCEF6~ujdN5)Ppuz7@n|-c$)cu8q05%YO<-p{sAhgp_0foxt$UZ=0d8cbTk{q z9I3as1am<5l2leUfwf;x2;YF>cD>$vB-$y#m@E>dWRNAU5Dae1T&9)%Z|2~GO8tDX&>Wbq8x<~S0gj&4RR!6vh`qO$@TC9hl`Nr5^_Sc#X`1~ zRHwmQUX=SNCaUi|NXS*}_vQYX?`DAQ0D}6aH;Wm*g3m+=UZHXPMnlcFCRN-6sB!4CEVXs|I1ajMAXOkMz*0xJ#GeI)0f#s4g(zQy#P%k!?kmhN^X8wg&fa zFBj8ij8kW(NK5UFxCf|8#NA-<9AlR(ZV99{wJXbFIbKSBC(RFI=c%!Qne$s4V~Tyl{vx$ThWgnZ6I?N+OukXIQH-Rqqj=7}Sn zie=>5A5QRALcE{h&FLwEZjWh76?DrK1~LMp(M^_my|{xTHC>KwliLMNH~@`9^Cq(h zkAKs2tC`}`+0!(w+?i4S0nrrvdo)_A?^wGCeguVC=oiR2+}@iffoTa|pg_+&s3(W0 zPFEBzv4i?AS>+YGGuj}W9Zp483~Atj<1-b zS+0|B)}`J`QAi1A2*Ez1xuU|G7>UBwa$SeHujy9^1#!_hkP> zPmJ%k>xF)^Fhk!MoC!wpAWr|1v4`Sg#c1la?a5bJyN^^sR;>}msCS&s;<5jf*txLy z!RvQQh*2<{*EK#4IdI2P^1F5}8Q}Z~ECW3yq)hZc$`<;eXY5Q>acZ-Z2(Y^zbE4W# zp2qL=?ks+x0ph!RSgDMMmC7(7?$7Zqglm_Vb^)0KI3&c__M(o*6)4rDBYe>C_;2QY zmOozJjP8W+F|1JW2)fa=o#J9S{Sobw|HheA7I+VOWTF8w9JlZAC5UbgRCunT3k}0L z>@*CFw!o4Vuw0s>acjWYsFR&+?4VVC$1f0JUtGgnfh&L*T*ee6oZFj&khN|X0>98M zgfa)yB+!7uXJim|Z|P(`F1$gT8+)B#E0NzC35CmaBk>oA|@HVA_^Ug^5*hSJXj zlyJI1gg*$P;L7TZ2nivo!mu_N^l-hZ{5l#pisHAvHc zo~szq=V-QhR$gJ#c5VgPs!bld*yN*sm*HxZtMwL1BJdvu zlIdFQr|IsAq3_Uvd$Uc8GL@GpiY&Y|7>ITdWw99StnXu4!q&A#(b#5%6LORCDsUTGGqh)7|cQ zxuH=D*?Lg&8ec%1D7mBkrC{Kg9!XQh@nLRFK9g$BAmaAk{?A z>)b!jzM%?Yk?i(aLLS{sIr*>000}R{&U@yHfGpy4C(~o%Ie?&Jz6Mi}b-(&5AYm-# z0!cWK>tZnP^>H8`c2{;~!JG_rwJ#Gw5A_%Ios%*RtN%a&zsqF$Nf8TqXPMpn5e27ZOORUA&UYkQ%(^r?s=nm zR*;WdYU8I=&aHStEunf*kWW&!9d%l(0hCy82_LbL-)VP)vTF2pEBDSRS-eNMhKF)%n7dd}_B8qBK@)`XNS6xh0 z7!z_4#F2c-UXR46p3hQbV{@>B`Z1+q%Gt;rt|&vhxLb-tkZ)(5ra1`!3F4}fqR~8D z29{_U&OvUWhCMiys>y*oQP^71@grEd$al0JJ5BDPBLczmc!Ov^Me0Ta? z`Q4;|5=h5IfN5t|U3(&0Fkw39xx3oLp55zLNZoqNgR=*rp>pA+kzMVsyZDRqR%O21 zjcwxGVoe=QU(s-;+Cnj*v$D3(iBQlZ?GMd|e7g#5E1yT^b^?Gj@k_k*_8j{u+uuhI|w2HrHcBRKaLP$f7)(T`WFUmxr3~|EY1tE(T9a#{PUiZE1 z9xklBupEge+D%Q%arVpw=?fyAYacy?!(mi_wBVTWfLU=JhP)cZP*B+Z9WNaazXZ1(3$P!>bUzaV<-J6rJp0xrxsPRlg78i^4EEAMM zY74DmyBs4v#M1#jK?oiKQ3@0)GcXT+a$;ULUT(v9Uc@RW!}s|Hx|LT z2`appps2K98bI39X-vST-8sOSoT|N%XB(m^zD3n@ZngJ4T8DDolXcc|Tz6w|SJp#Z zu&rya=HgU=(L+dwqLE9pE;D;v4mg5vH8yEcs-(jlQ&+J((GKXr00m)Olw2#VFNr;;biNLjyRLXfzHl!mj`%74HceFmyQ-~P;}Bj zBVA+!7+kxo!+DsC>mxZPeg=Mx!!RNcN?NFv zBSEI1c|NQ-uG-e9GGB}fLtC#4a?z=^MAzywA^HEa_if#6OEy+&Ozy5rM3t2@1;6Lkmu_cI zia}&JgU{)I!@D0~1GPD7a&~8`mI)IyUbMq^Ea;^1Ky)i|`>IZb68ZUE?o?L9;u9;p3xpOf`oM$lZL$jf>^eHkoGY>C+swcJ$Ok znr%>q2Xj&}Il+c257h%*P6=+HQWP{weuk>5P1P|9OTuVZX4e_g1Z=Kx#4dMO+WXVW zevox+O&Ge+M%7(dT0$w!SVD-!(moU;P;nt#AbFZ^l~2BkIV2jsHKnuh@^#5l_oKo> z$k>H22!rNW3FvV!Dx#%py*VTXb=f*&vWVrCY$$`ZfecPcW8ot`1HB{2>AX#dsyxdn zD(MvX32&T^S3X~>dxdo6Pxyq+>D;lQ$O7YV;m>!b0T)=tL{|lwEktd3Bb|!LVzJy(T_F)B zlJ5F4^AJmPGPAPI)P96;5!DXo*VNNJF2Zfj*5qx)B(Ewhrp|_Cwcm`x?7qMXHce-0!-aH z3a8?N$0>!$<^t@bx8uepE*xuJs5K<{+M<9$H!g)d5{aQI$-Fg$QE@eBmT$Zp5&2SL z3o+7$Sek?$%NkBwB=2O*ww-KvEA(xp6;u-OlnAh?Z>DP_qM9U*Y#wvK~^&KRJlDML7`7pHCNJkj25ad-*JV= zojC4yej7#osL(GMS{5%9!89Qj%wtrPK6#hWh&jU)+)jY?~L_J6oTU z5nTkqk1G6&Cx|br&Mf)eEeeLy7oZ091!W0S#E~xl&W8}3VCb%UA8+oK>+h2_F05}7 z)xxz<&%rV6r7;VUVw-@&Uevd$Uj-Zj1q@%AYC5kr!$&SUtZNt*Se4_#gW-S&L<)ok zQnPW>33Wk8*}k?R$b6$rKo?u3zq9DXiB|sa(ak65FS){ffQy88tsr7WdmtX{`01b% zm7|b#k$|m~EEiqV_`q&~#>$ z7{p=o6WXNCet0Y1;QHBny%jBQ4dr{Qh@crqi3)FrTSqSrvsZzuijC$}umhhKqFj9JiUZ@ww+hbJ-Hzu4!&`FQLNO{+0)f-tPsX)`gB?*(1btoVK9>B#} zx?MpTc33t1v4*M_Re-GqLh}X;q)XP_e0FrqA+fKIb*F}R`rgxfZvBHasx5QkPt!Cr zsSAG(x0iKgC{m9>Zo>sMosug-+5pW@P#8wR#0j{%>ndH)6jgL{h>Pnk?>pc7`wc>+ z>Ak-O9-n@;$>}ZO?LM39_Lww+JqtP|Tosh>lZyAfNv#^Qggqhoc2wl5b z#hD1SQIw!Ik6>rO0+@28JD_J?r+kM#;e^{u_O^R*k<7iJ$03p##CsOW9;f$gNkJh$f7a}s98>sO5(ND)4tOo$Iya6lYI6I$52Sf9yI1t1r40Q>_ zwoq|YQKYMzkGY3E5KJO-rlJ2Kx1od%`th3DiJ6*C4LqaZfLa545AYN_1f8&`XkleV z0qf)E#}gnAbAZZy;88A95e-oA;Uf+p9T9ToJgw#^qq~@8Q*?~~_{V02s_;bUTA`%B zbqAt23*wB$;`!wlB)y#@La+&{sFc$)jU-b}VMNA-L@%W+4D&QLQBu(rO92@;40_Y( zNsuCWK#V@m@K1gN4HEJ;?-aOjKq$^6S0WUM_K@(+azjETSQKl!oy;eTDUt>5P^HlL z$qOG+emAaQgH&H}q)XmXabFo{JRl#Q`}(dmrHr>x7yGeGqGPq1tf^GZ6A};ZSbnD1 z5S51wUuA*>$$5H*d}q|-XR+Zh)^0`S1@-<@$W;TaRU%{y{Bz=h8%K?_&h8(!@Ly6+ zWyhi1=mGVjBELaVF>)o%@DS}(M<#%FYYZz(h(!RWtN{sv3BFO_W>m3Lh$+PYm#*6YbnvLToi z4yP3v5!X}z_&;+hIFP{Qpq0Glvvuu)HN5)I+_T)IbTzIKhEDg4A=@7h5qGncbM9@zd{}hmO%*&M}W>N=xHN3 zhv@u3{2>8P+`xND%rezMViu-R?Iq$kbO1#Pd7zrN!0i^G1LrkmYuMKeo#fao_WIDnztfelTO2%@27bF~d=xwj)ER*Y;^n_o+dqt1_8;@eI zRDwWhw=lqGk+@QEV8}%%=aye3B4wsAf(A)k3Vw&BYI?k;B}2#}QjYu)`usCcVAx*? z14Ur#s&lT%7^BzbnpiWa?9Y0Q2oCt4`{Q6jjc&wkufXv}oRx7#h8rvYO1^-S7z6u9 zoQ}3h)UT0%y-C@}Ai|_ah}gbc95-$|CU-cZ(y^l;V~S--tu7?=c0I|q8}di)YwLDf zl(T}&XO=YeSumu!;wO7zw^nnk(MI5sEomd;U_llmGxxP~yXe45^jv83D}` za*rr~%gqQXys$zK8SAEuvrv8xny3etw(_(mWO~rhPv_uxzsX4ZezeGLEac4=bv4p0 zOwya}JIdien&911QTH)0mBB#6ugYY2X`F(N2Pr6h!oGP7p^wn=`vGWp$nDxt{(o~= zLvc}|cIYcr*W^_92gXFse{&#uLD=c>AWU93IA~s4uJXzF=%JqIfcAK>Ap0reP(l9B zww(UL>G+VLwzzi4)_M}K~uOV z8s&I1&T7Q*?Pj%zz%WdgwvpjuJvqC@f{%68-staZ6*k|fxU1-Z>9rV0Zw<77*rHZo5P>CSh{-m6I-D!!I zfNxDb3zXY=T5Lt1q*JtKDA$^r;c-z2;G>$QH`0UT()># z{+TB0Vu_>YkJml0q((tqm_1;cj$Rw5aJm{$pO@JTU0b$K>qYW?Jz1^L|MXc^r3Ocu zB22-S(#uL3sQ~I83$cG$`XOHi1M&+^ig* z>$+NW6$gt$G883?G^dyD7V{^6E&#G_WK|eeR>2lfnP)SQ7PKs;7Nb4~J{z3cb~DuV ziaNKRW}|>nH|5bqAU+Kcq(s$n1qe6d<^&S-HAh_slC3G>50h z?V#!ffT|P{Z?!XIh8C?+JbllK-Kw0nc9VtLpB0nTbU>>oH(WjFr3;#z8N}KB_3Nw8(b2cjLtC`HwMpDG6V; zonuU*x>z@2sjHerESY-_gbJi_Cn{q)dQZwsN509FtJ*i!ChB^U;Yv`SA5qru@f3tJ z{HU0v1dxhF<&UTbRAnf(C%3vZk*Zb- z*|J&9v#t4%DuYoAN|{JreyGBEXs8PCxq$70)Vh0et#(M&&U{#-Y3Mv%+;1O}biH0O z%R(mTHfZx8F%QCafI9*jHRYmXv=u=H6%1DuT)M`R0CwAs_oo$FBJZV1pHiJ$aSS(G zDNcisV&MRvjRNHpM`80c->M{!DkkAod~3?U;bkf{#J#D($?7U@)HVhvZyE$4K_B*Q zHmRd82ab6CQ4WkII^w$F8R!5(uSK@aCiCp?6zv}8^W~IUjbjtkZ@d(%Jfc;yLq%1p zaQ%G!HG#H;9B2Gv%=(=82%`=1=a%84DbmH{mTE-bjsBQSHtW*?>eZ_HaJjym;d^Dp zUBH}sN;m7lrF}-%kfQ-~{|f!Y@HwxRe33)dg{cr1o-9zu2SsX=Mf$y1j&NvqWY*=h z;BLqCVv18(x`kH%jfTzg2jf}y+R!1_b<|jN@pnq;=N$~54 zhOPTh&SwqLh^PjZ=c>mSXFr53V>bp2r*30Z%}%v39m?~?0lie4q$$L@TA!`}2f0(O z)kj~r)3F)QEa zxF!8w6SkcfYKjoAjaNnH6uWY7UG6XEh1}bW>$Lwyu|FMK%r)U)=IJXuM7!Pgc!db@ zhJ%hyI&ytd*I;vt{hew8#*qpUWhB#ur4n6rV3La~ceD9!vPH5ak7^#rE-ye=C-e&yi0;TzB^;{)h26YPhkTFgi>04Cc2tmV_d0@cPG{_WjObPG zad%^{y6p0Atb_MyiN#|}T-yA9<3u=f_GX+MojT5iwNo809Mdo?3T;ymn+Sd|rv>>} zm3TL$3wvOO9sEhHEfNn3$`LBI@WZyj@>GIjqt-^15D${bT9idRZ1`o3MC$v)!gc%7 z1$IC5Z%rIhXnDs2ez)q02T55->ek}J#^jHNjdm=mc#yl$tiqzu0K!tF&=hi}U6nS+ zp&$5Eq!xaO{`2ouHAhpCR3kcA&9Mg}WH*26+zU43DvLWdhiMP!q*{;1iB4O5u6o=K zTr2eheiVEnzz>fD{v*eKyK<4unN8z7u-&y%T7umLhdOi=WE8cKWDmQYQ*j*owd7a!ME;*K1@&WQN29-KDyCgbc|=PWi5@=y{hcvg& zypMSU{vo`Nt(Xc0I#RZZ0Lf9Ps13OJv8~9vM=g%a9=s~5=B0M492Q`z1g7x!{1Z63 z+>8aY;19$5Xyaz9K)IRH8!Lx@| zb%^z72_q|OxvP2h0B?s~_y~YIR`wjJ%AOyUA#flxF4^xM-;SrPA^lGlu7PaPpa@h$T9v7;cHZ0z|S>Yy}0|0f_Xzk ztm$(KYn##<#CEuLAw^fvdt`|8$XD;6%PUQnjklq@!_{9SIVVIvE#!}ATBUM+n}^97 z70*`Nbz+JnkNoe;OPe4-&;sh4p-|f7KJ^kslxg|^o$mIb(B9wadTH^Hp@eFoV6qBT zg39ZhW_}C;VO2eribdvyu+IF2umZzY2Sr7!L0eB>hn=EJY*g+}{dgQz*WesuFca>C z8=RZ+g~@h3$+nvj)xqm9KWB-$)+g$HpX6gNi_HXAyWj|6k;QIPlY39q7pEC#sVxA> zW*noenOHRHM4ApSuIDML7zqXY>_!s@gre@$ZC(VWwjvLO-cEy$y1LRMqcGA@6h#dt!AuZ0F7Akal|1??8 zi1v3rk%Ad{<ZPqX@+CHVr>YChdtYDx7e8Z zHJThi+E@5;9!+zi9Vu%(;3?a6nyQ4iygV&+I2>i*yXUJ(wjL1&w)A&}Z5U-)r4bL6 z!q-1|m?G7WsHP9k#8Zj}Fpn!3p`#3*1{!mYUL+%opjtk`BXR*fTUDt&zsnXg`i6#} zMvDL}$+qd5N_EpSsy8BxI`ze3IYaepeE2*ZVrAE#Br&0iGJ%l4?i`aoY_{q%=c=_^ zm)7k7?H3E0ltl9K*&*Sdx_!)--&1rxK=aC-b^;D1g%P`p>0$<9iVahrnnkNmbma-O z3neVF1HRy#kh07jB+-2P>WJ2{eK}zrv@-Urb#Qx2!lGWBfM&$L1GZ*- z+k{c@GVll$sU1n8bC)E^Do@%A0SdKh$0ekHF?uGx&56HG_z`)Y8Z%s89hfoiYk5p_`({sbGFF6k zQt2<^0xQ2aUXO~pGKHZXfkSQ^t!WdX(*aBo!k{Ki#u78j62#_sS6brb?j|n8=@v|4 zKUwpn1}&Iq70QZ$A;LV7ty(V0c!pf0BNia|4y=v6 zksylKGKogRe{F-LHwnT+Oo6U6NW!x~M!bed6t1c?L^{i%s6UK!xN|2+i}pt@QZB_c z8}W^>L#)vdem9iyZ!|fEHEoNyhouovb>q>Ic$+P@>HQih4N4=KEViRtRZNE4hL#)R z1+9BIbS&K&7-#5lg7(;C1ToE*dcZQ0Y&;%`4s=rmPs#2VkLhBYpcB{AS_-Z2!8cS)Uv8pO# zBiF7+1*+BX1lRWt;?trI$hCF>C)y?@K1)dv%73QLSK%M4aj&t{;v0dEZ}0vEq3_d- z6}JKwf?nbv76r25_Ad%E!l4u~U9h%*LMJ*Tml7drovKEJ|n5bXr8B) zZ@X1fWpZt6Jaf501YLEGl7|~S1|blU;8Hx9{@9izoc6QYvr*@L)e~TJO8hRh5GU(` zfPKt_gbP7h0o1ab1psZj-0~nVdV+$GTkdlX}Yw3%#9VlUC82V-t(4x+`VJ z@1SE7z{PQF&SE(>38Hw8O&SgVZS2^@vqS>XI=#1ZY&s4n(P$UPrt|#Sa;E0|HDM)T zg`?34@{9QvaMU;=xddwjRNZ*Q_S!l&>n(->%&K*#ciiSy!%j?-A5m`D{d{?gIMp-s+ia0-Vd%L& z{-1&Gi&NhhXZYe@pTGS2%V+4)Pq)b^S((ef|1({!@a2C^OI=F@>ggT4SuL0IWchu8+oyB`b3@r?;$AEz7&H7M@0YT8NmiaBojsc- zn=SIs??+SmXfVY^R(K-SUBW3TR!FVF|7$n@=fU}ml9rRZ_41M4%O`6#7KbArw7}|M zaPtVn{$F+JVw8P;9Gn0EwV2WZ-VcyTeF|`eB>RN5!G{Z%=C29TC(fO*$Ko1Iqdk1L zdfGf7d3MSZ^vi=^QawKkPL6(iE|yckY>9-dywQK%dzPnu@3>#MX0A{>Jo`V%iNXd^ z^58y0fG)xMUM-w!l|9 z%c-ME@7?A&SCA#M$#z0ltucPSn`S_EYIzQ(kGLxYc}p;tPlj^Vf2I3*<4-clqPno^ zrpx1JI0M(PIAKBfbIys*yY_^?x9s;mmS`C1Y2{}6x9=Ck`S^eoEPE3(_j!O1e&tkDtIU^ zA(WJ+BYDWVb)AGCfO3kcL5`9Z%IIlZW%P2EK}A|j*(Dwf z;ZkR{7y80pF0z_A?r;w-X4&0cx=t}nLP|Lm7LtTHxI`Htm`oY+=>CR%FkP;Gvb+HI zjAN*D$V~u|zby}km`9I*1Dw-0Ym{+C zP3N=i-6}b!d%8pY$BFsN*gV2wpsVnt@IjHql11h@1$ctkBQ@P&e_*V!`igwO^Y4?N zo8)dX2Q+FYbrvF_m+~f|Ohc_HQ>YoK zvxq0W8o_fhh5M4G4M)|41!v=^GrgSm5s zIYqNUzh`LZHW*{a+r&_JRplCZOe>L`&}vF?aDAS2Bs_}U(*6z-Kp?9?$b#LMlt7K5v%XU;LKz|HQvsP9V&!fcmT zP=gq9xv>GMs8g$}RPO}$Z;UtNJPUO`P-1tcQ#Anv}P`ss)bHp=5@5qCL#nYfG5QF(Wq;_{B_0#zR|VzzG2v7=vj z>`2RFS;(Eqlm?)nvS-_9BVH+@SHjelvK-dAjkz5QKCi$kN9dyj4QL3gqL9)FKUn)l zZPhq1`~;b$we|QKY(ACbpryJ3YBix|6}rT$!W83CvR#f7TX)|qK^&)=?H5Y|buX5< z$@VnoRg@bbojXU`Vjrcq*wGsz5-Pmqj6R7!vjYf%(f#b|1KYcD1QmH!)z$0RCWYD= z<5ceuDFOoW`Z0;d7S;g6yRJZj<-sH_R^ANJ`#B`a>1?UPRb5fX&51@lSI8OEHSp_} z!1rkQ1)KCmRsmys<3ZIPwyYJ=arP1*x@J-K&gX1b=Bv_vuoc5$-3}5JdigZlzCG-d zfJ4Po98Bt;bF>&Gx{elS5~M;YFF<2Pet^7!FQ|n`dV!VX-aw;qImooZad`>APoy9D zcu%2(1_Qkqoy-w3cMo4wx6nNf`=ICU zVh1@Lh1Nucd#EZ-p2=Mp#e!imRsA?Dt-3q3logUMIl*R7Z#$w(KBzk-i zp}xUMuwPtL?+nLFTkjE4s0#0q%BX5)K4LclJB3kiu9U}A`c0^&J?=J)@g-3!c*rbW z;T9K_ic?kfX;|od1zPaHJVkZ7WP|GW6I8VSUkv*Pxq>M6eZO4Jk~Qk+Bj9?c8L$*0 z#Xvs8vS*FGcl@4#OH_Env2s0>0@K2@vZ7-%xrB!dYEH#P_D?wbZ`U}yNa^(M=IcM6 z-t@QHo14TSyLlM)E8!gZJ0r-sA}vUFt+f%=V2OnzD zkM^Z_ADx=UkVM`nhnglcRu12XBMi=0v7-7t`UmKd90}6N`1%*`Z8>k0%2y`3#AS2x zmGDqeZ4!THC1k(y*>e|`BsDFW7xOtf1=_Q=F|r8xy6Db8O+WOG79dyH_T>>o`g`7e zjcUzgR_~%lIYm67Mc_=ac;s%SL`t zaXb6y)WV7q1>I2exqN|xDH1(fU2@dTkc#V(SynzR1^k5fwjtI?^8rv4BXGl46(Fy z`?OxDGNbV9jhr__6*)^0P8xvr(3}@)Ms1>9(aj1Q!`Y0!IvnMnN4oz>G^jQail9Ql^4zZ znrBrfeqhZ0`EvRvVdvuUX`BAQD)sG;({o!C6FrJhX|foMZc_*{DRwnf-OpUY$cplT zkhy0`*5aO!J=EFF`y|USQu0J3C(1dP%|3p3LO3H$bLsg1sp17)Lo^XOvj8@HK zB0JS<`vDAI+=2*LlORKZU50WtHrJcV`@)aL)k2Fyx*!B(%Mv)0Pf@xNEFW-<yS` z>u!`?bumqVFiQu-mT}PZB%rR{g`tB1zAIP{HLcv@NN%#s9M`i1SwN>k3!Z>?KWkYOP92XbF!a3rw*yoMZG8^*6Q&VxpoSgO~+oVRQgD;k**F+fr*SSw$zA?^XUv^C9ep&4?3X2oUsk|Yo z89qP~i>(KDpKO;X@29W5{8ZNDySsEdeV{(Ug?TpoNlftc5uG*HKa=Ge8naU9K40qL zvzcc8C1uI6F-;KadW~vRGkElLx!tLyCyIcNK`@l_L(1Y!!=mS4TPMlI2jv29hVk>J z(13YFg2t2_EUkD*rV~FOR8JiL@){GR!iAQK92j?1pB9P|QGt^zP%VlcCYy(m7DbooUr)$9-Y(7GA<@KfC``}6S;SegA7%X|>hRSq zgfZnXxTGsHiovBhoG1wnAg&`RJbFQd>SQhDR$}&Z`YlzCF>&0$*`kp#Cb#)*?*BqgNB~(Wd+tM5yH~+q>R|P}?^Kuxuw3o$=Bhy2rswR4Z z`l=1WlZ5ofVF&Csk^voi6pXS>E`s|@q2$-n?e`$}r~7A;SJcBBZsmx!T}M~GyC zvvnSMeA(_;YHc)VF2qNG@994yeeylqJ|K$lH@nO;8k03}3E%OvYPG|2o7zo+h+V!C!`4EBld6O82P=_Z@b7A| ze8v_kgbJ2}I*D+&;U8?N)8yJV4U!F5$cyo`P}GM~_xAz`l|eD0tQxNeCsZ+tJVPo} zXc0;FxMKR`NJ^y2q)k`yP95ykBwI6j8CQmrYamq@dt@fzE z#as|xP#=>@SRhE9?vc-EWEInNmLremBo>^Et9_KXNIpFT+F+fa6XOcu8X{1=3*@>Z zih;tSpHNqOA=rz!tv(+`DXuDoXxm`jcrEn>JEs2-SuBqDa#y zAb{6gSV;T8gU3IiRtAJ8Ws#?X;XVuA^JsI1XYSY!QAt)TKrB>(Bo zupSb9PQ97hpz)PBby4a&27EMBLt@P*H?a5hS>E7c@~17Be5B=f<>is4-y6#&c$GHp z^yCt0lDlB^iJmLC^V{y+nFjJPV&VRc!HC)FygV0cf$V9?b7^2Fi!E8Gc#f1{k~+#9 zW^#us)NkjLCse{>Vu|s0g)!wG7lMl1*B#!^&vd%lm{g|4N!X(o;e~~2~WJWAK_V26EUgQvrAU_&bp)1H=xgh z=!sdQOg0{-)dBIXTJu!zskw};a)#kJq@xssbf4!d2uM4P)jJfvaI3NHf_NcS)_K!1 zzw)XrG~-qtloHzKpKnhP+@3EdJeQMTuBNivG+63r@#BZhiW>DTKZ0ZHM$5%$GDs6o6ES<4vy?QAVq0*}t*$M)RMwltZ?HaukPPA<_ zL|uAoqd{sK&59dFRpP5My}I%6MzglHfn!o;l626bvq%~=7K5a*;cQKmeJ{J=o81y~}Y-EqZaf7IrZP{B48bIs6%0hjqy zAC8jcLObBN!>S`TbD86#p0i`xG@Ft7)GSzssdb+$N~Dyc#J;)%INSrdD>%K;1qV{* zjrL9#fqT!r&mB_KjRx(wlD|7q3}uENzi4LecSoNi!pW*$fi^f&3U^^;IBwjIekxM~ zY^3yT$ahzT&wv{7a0HK=QX5@3V_=o_f3)1n-YEgkkq zwM^dAKO@0u!^`DNRNHeh`NXY825o(95FzUX&VW>dH7bO3|1??8C^E@L$6wU%r>L=F zEx-CQDavo9)1?VZ912PXr(>XPC`-`$ga(HPMK*8=^4@gCs6-G^)y5fGhHh_PR;VOXkXpuuoFqS;vw~ zXmf;Zm48=jiAOhOQK?;J#zY?yFL!i_f0H}dtyt`fSGs!jU6&GN`oIKJen-lg z6!h-L1xb}maxk#O`RD-!F3by0-yVtJaq^>l@eeehS^6XH#z^l}U4N(Un<0dnIW%25 zFm;Cjhf$Fu_;QpUN!=lF;38rdqo{DfHz4~Ab=Kp=qA3g_;u)wRncvOxl>lU_a-UjF zB>br*UM4s2x=ijoM9H6uLk>=x2njXV74%M z@SZB-`FN9Sr+GvNCD$BI%b1wWx6-TQ#GI`ZP>DM9iI9IWho<}HumS&0*GpsV^)?l? zv6E9J-_yyTz0H?zKUXgMJjwZlO^!5EV(JgC`{)ju9NoN;VM}iX-ZMux_f}M*sI`E< zAFvwMR!A718H?_maY>_mD^&dJ3~n%Z$571A{b;H_8%$BG)@tGa@e29S7Sy+> zbF3Ut^^lz&DrA=_ylu&J^`z{Py2{a_V6t+rz>cwZxofeY=emzE*QeUa&KHx%)SUL( zhjVUP|DozS5UhHB1aAw!7A!I8p2=M(5>PZd0{M*NPv0Ih zRajh58;&4K2;|1Ec6v5FUxQPsAD*n9Zs*zNp^9}{k)&O1VUTh`+J@@4qTQ_e6MDSJ z*VFA-KB#b&Hdd>=!dYP4Hg&;*8}F!BeF*QU5IfxC#c(=)5S~IoFQvzP)-Xx%+Kb_s zM{4h8rcxe^o6h`M2F+ft8nj}aVJwoegZg^NG@7T2d(^~9*X!lFq9-?kBvh=lk}5HN zC?)gQ<4x&2=iEJ5vjFT*yNn)8r0b0^3T+Br&NsE+tTz8j5r)IKCO=H!axlh^L9!%F z$ID`Sn?kOWiv(gR;%@klN@1hs=FE9tidm?o*g(B`jp7?9b^0{l(!&mHd`*ULO)Lv1 zB;ahr=(SFwpf85NgT+H&APNy^$_6#L+>}jrdmcZ;@;xY+jJKfC2@-|uf;Ilmc1a9V za4FZ1%?*2-Sf$1gHlmW-B~>-X_HGJxFfwSC_If3gjGiLl9W9lN0$Y_-GE}-U8Q;y2 z3zSV3izT(>kPW^~*=w_OJSMUZJ)Ej8C(ZMkCoQcFKk=lI6H^;wS7rFG;cEOss2x^n zveT^g6njuQ_HQY+e!nK|p32Bg(1+z#Wi;DnDk|(!=jtKSbya`_Vx@0|Ne{`f?aj5~ z)SxPxFV2kVJljd{2p>6JXb4%h`!QloBq3N<20^ zPLn)tJ+?PZ(XG*);?Vh0x%Ggwx#i$-4X}gcy{(S;6-E0t6ICTqxKXJQ@ml`RCW+YEBy#kH1^?RSrZ z)pd-@=_}QmYL6`ch_xO^r1fP~1MCSs2Zz^1rKT(!f0~ltN;ur8E*E!ko%2%+O9DCs z*A)H9!8rfLIK`}iEOuz-aHG?Ex<+WYM&~IRu(%vYO=wJ8L@E)64uBi-8{p;^Mmim= zvUHk$&rm{L>AO@mIoT37yx2ai=IQ7oMOCq;tDAo(qmQ&Y`W-Q2JW@>^1$>UAcU_AP z_xNrpf5#yn2XF3{>+i^LOeSatLGu1X5RU7bLoTQRf>)$IACS zUldi@0)4lTf2~RxZuPa3$_@u;0y~fW$3LOP1u;e_lsV`4#B-iNbz#g37D}bl@pI-+ z0S@nF?~oxRLq1N%qJ%c_!q9omqAIg)gDHB}b!196ndFTTO%L>{rN+PwC6(6Mf)_)W zaISXBH)#A8U zDG3?^m#Or0sj#`|pM#7^v>C{IGVFI|)eSpgV}t1ho~i~*YyJqQG`#~o%oWr& z)EjjRY7$B)(KH-ECwPU**>cgY_&XdlA67E>&_5t>PrQ98ImANl)wrq+n=6*9y=(Og$!Ko#y&LYWX;x?Ou{3pk^lDzZdRz*!(j_q zP3ax*H2f>~QL!sPKJim3OKg0JZLFEPJjGI*6~eQoFc z`1ui}Q8Pbfzgb7E0RSIy2#9Ht^-WGyG_9Lu({wZX_{U}is{sIFapc#fzjbbBLx}m= zwU~N&h^5nLL>T<|1sKq{r`^VnD$sSN5n_bli1b&9O6t&Nd$e6!v#2m>OGlQB6+I$c zLw9OoIj=hB5mq^fGqUZUwb9>)rokd4x*iH>UY#~gBJ-ZhVIwu!k$Ab4G@K0jeQ5WF7g zMG4-y7{0ZQ2|m(db|!eh5hnP+p^dYK1~sAjhXF#bwpLopqn~4z(?>5y-_8^``hEdR1G4ZCX=K4GwMWSV=O~r%_3nQ;tFIY z2$>_N)Pzce*;5pb9aUcxPK#1&sYunB)G6lcVhN+=D7*K44ng4kLQ7Xwqb*9LX{5|9 z7+zr@wQ)uk>PPwyBpB2}6^&80lrgOms;;E%0&=Fq@Un(s@u)S!-f6Q@>6f%{QOjUA z$OU>{4Dyg=#L1V+iR5uyc7U4{zVq^Be>J>P46o4{X@5RZr-@wP{t|hvs(###@>cT` zbmEs-kt`pbXGca6`At=(j7ro=^2k1R&%vi`-Ba!n@r^*|QU?E&o=zXzIaNv!&4k!h zDFuNZt85}3g}H_dG(}Iw-5qWIy3OrVE9D z#nSTb9lJqgtU_w-m&Vw0p2(;X7mY$Wf2G4)-4S_0^z?zmXgZ0HlRx27L@2nB-YB2V zIW`^=+0k6k58+-7J}b~7p5>Kjcgku5A@2sPt?TT_3@Zg23Qcjh4MZ z(K!XEsdG!{!)R%ta%VUPa$8IthX)~J4l#U1m=N`nI1Z)Ygi~lpBI%47={T#&nDM+8 zI%o(Ct3Jn+m_0#BO3~;-9Jc_$#&J4?Qp%|n2pCI)K#6W@a)lyQMLxMvsC=BsE61EU z<>us6d7OhalNW$K20`R$zodX_#lo9U(L1V10b*(0ptdea-6%Ax67CtF?H_W?&^344 zNg83lg4rt~J)kV#!~D#s)a|!O-TqzKWNlKnW3>zgLSkfS4-*_2!PirO+$-u zx;!(SO?)Y%81y*v3F8sa7kRTg=SW9`p=&C%OHt8ykxP%L2Q?TTn61%+I=A3vDZqsM zE_#srwHSGMit<6^_;IBHQr$J43K)TcMgc%zG zYuPquCPjvoxK-$F?ZUB>o#X7fpM`zcV0SFMd1GO7j!n6t*XleIxtXF--*Syqljr77PJDyU{V; z%Vs4!6?glpFwmg+n*b_!1@cjfh&OQ+`yh`Uf#=E+>_uQv(e$W zfm?@f4OO+*_2PMAJA<%CPpmzBLs^`!ru$F7aAzZo{d;p9{ua>t7iBI`9jm|R4 z{Hkc{!{lXljTmVQI#CDO!duJO)#Yz~vo!`dOv`9j(H;3yWjWL09$^?s{!r_lj?KPq zx`xrvCmnh_$VA&jZX`A5i0#XfDcQ{KhL)PV49=bw%k?Z>qXHu;cThXwc`&-< zrRr#b29oA)=KqH-2$NSuv7uLrE2jA7Yl51@;-buMGr4nYY;M7GI^C*EMyx0RR>Gjw zcN{$3o!Sl)knfy}kH$rkzP@zm?fmpk;z}a4G>V__gRgmMdRH1at)~Ian0B1#jSl_# zW_zxZ9F%yXk`y$mVk1(WYYi>`EX#$Ieh`%Z2^~+0y()F2UrBT+{A} zQr%a(jg&HH_EDbGN5xeb!@^~39+>wLPaT!S^T{jq0T2&V|HwVW0}=C z#Kuso+=c_;gdyrZ2N`U6-XanzRS6%Wxs69r!G2W8wH)DQ=(=H12p;99{Xa#U?{I=Plv8 zQ2;d@>efQH{$|pnE6_ZquFj%caD#y;Lyj*XM0uWmz^o)4BEx6m&S-5`3+oU39hsal=}s&KWeL{Mh`K%P z=C47V<41k;>S4N^PF9m?=IG~J|E}9asy`DrTNI7S>w`FI>NEwP*kX>-&BHod{K-(t z`A;L1zEx#@maMt)=rO;VtT+Wz9>n3QzBwH8>BYmIowK`USP+DzxkfrmtI~lu(R1!2 zMP?A4J=Tz&`I+k2lVD%i--@bfRwM>|!kK@Oi?O-sOXZH(Iq0yTbQ$r*O zO{(EYj#uqba~WZrxp8#1vt1k`S6Kld+r@Tq&C$T*XwDa16F}KE6q_4hQEi5Xb9%*D z^wmYtf}A$-KA@=R)h-v zdsW*9{xT)`a=&`g5l{=$)!F0x$N6-*PA?|w_2g&CouG?}z(hc;wg1dvC=3%^&e7tj z^`x@Xe0Aaz?#KWhur`&pI=uSN+_U)PL#eqG$YuKDhut^FUX4&x@2g5}A<|;cDk@Wz zv+0^S)$ep}YVs@mTz@9FKdA%c^7b3zBXA_Ge$uYFzMEy6)e;&ReVqVh)h!&n12JJ! zl`m@cZ*1)x`%3i&6~Y7w+^EJqZmOH|<@qfUTCEN@CeQ8J+YK}x ztQc-6xa_u*Y=K-Owfx+J2VyNw26b$I)@mMpgHiAFDZ1e$R+cTYZSp;X%Vd-N%^Y>3 zG4=;(4bYZUx7h9qOl59evdba53>X3Im}Yxdh8A+#Kr72NkH!k&bVS(qYGUPSg!3FlTQc;a8-DmYop06{ zJ%5(|KHF?(Lzth_IS#1`u0Sq`RZ^bQI?cn2Y(1D8(31$shg8+1OVg`({zW2vfbUf7 zh&r%T24t9*V?!Jv*I+SCM^CDz?sTH8)!bY?Pxu!kMzkS29Dc#(f0227dTaF(M3js0 zmTwAgM6YF)e2>v^-FqZL9BTzy5PJr_O0hwq8^`kijG+glt>pMQAy?8j+2%@UshaYP zHn=;I&QT@c7J zlt-u1(3Y||gBbpm~t5Xkq9 z2GeW8IDmx?EFluiK)Om8r*KmHwIRL&FdC*fN_FIZ5g_+BAa~K8SduOCjgQFV*NNV1 zUSumFo~4-rmaI&Se3b=ehX8`1tnPJ9Ch~PTLLWlQYyJuq)};c21HV6g87szU*`&i* zTdPP>r%!^GSApUlp{Ew~WC{JIY%thG&IU5f4cgi=%To~?<{%o$f)!~9-Vy{pQ%nd7 z>2p6S*Kw3h#v&WFI z*Rxkc9%3<|`^6G{k)GyU)s4CX!e?T)0v9X;=X8i!4iDPg0?}W9IjPD&I^~JPndw$Qmu~ z43S>b%Hb}B*o5{=pMM4l45KU|q9|E%r8H&bctUC=MH~|fHBzSFDNH&T(|S2jMwb5~|SmdZC8)eEAAYQE)qPUcP`2egR!E9#^?{ z{fduF+rnC}7!_W>=bnYZPGX#EZ%V?u#W=+PYI9!qp|k8L`Pgzx8hatxy|ut3Eyvf# zc3Ybx7n9_j_5x8J=V!sQAjwQxj3e=8A3G@+vsFWDl$lE~W~t1;y+y!Q{8~B?Wn}FU z9Z1J63PP<He`-EII%J*|Kf|ElSzelhKCWBaLQpZ5>gG-zHT;#_e-6E!Qu*l!E@o_{4_sw>) zM)zhU0N!lhkpmD(aCb{Z`A4*h!9XuVxiGz?bLisMr#Z9^hxC#9`hEcV);)6%1D?|y zp3=SiO5U4FT`ajbMc#8bZBk^iez_5E9s_&*AZxO6kPrMs1eCW!hx}-9{-13*%l(K5 zhJd%2cZjb8CmQnaK^7O`*COAiB-4uMoT{R;SF^aI6B7=sfG>z#o}YB!mNV+YzwoJZ zxbiAqxH7&oT$qb}W7Zj(Uok~4V-FX z3Fp0sYWwp8OLm0A<@K8R)o!ukBWEez=x|N8z>zPTn*@84QQ8N70Njn)ss8}(##Za$ zZiQ>`z!l5}ocegy;@y|lLcb?>2}I#x`Ac$u@>>Y6SuT6;hrDjl?7Y`w@e^@y$v@_#Ug9(mmmJ}Jf(#w^MDr1ub?xPU?1z+B@@rHsxdCW@D&&fvRdzZSj6^Fybu0@;0)Dx~)^Qs3&>l+p;DxSuu&#-Uk{DXW(P{6; zyqMRtGme^pyaXjb?*~p}LaEC08H(&XT?8mu7p~XQ^T+Gf7FWp4D&(%c3LL!cN#&xb zvc-qh7kPP>uALi*pO@JT^$fO8>xHV%ze?AmXLbJ_ylaXiJhq}f7u3kdudXPN*uaHg zmV}hAhQa#Mxc{h;PZdqvHB2@u$LFf96yo>)raeQh6)mzg5p_4)Kl!h#B8>+gd7l4b zlLMe|=*kV$0tA0BtsBJ-K!6wN#w^3$82vR-Yx30(pp91dCZC5Vtjb(#xO${Q zszgW7mD3#?Uw*_|?hARbWd?a z6WO7F+<9I=?wkt9iDaL~hoPpH9E!Y<7}&LzTnL#coj4p;t8?N*yj4?;h3>Y?0!_^q z-lD=>2z*`q4e~Cp^Jhv+qu87udw4$3tpxKmS%(*Mq{T(+;SpNA7Xojvt!6`aUAf@x zAYG=C^gT1T66DOC3)2bIQ;j-Q*;4hdphEH*b+YnWphle-5Tnr0cW2Bxcp#Nbux`_9^4R9d}|3dR1& zDe5YGbt!14Dus}?Nctf^M0NG)s21Ny-A5SZud|a-^E!$3`gTyK?~-416Q$+)L#lc| zT~qbN>-0HYqr)=L!!%rhLn1q(>~6wj$BA*iI4V)u1A>|kuqcGuT2*lN)Bv%d?>%PJ z*NdNNuCd3?3`?FJ7p<@zMF7PBWCuAEV|uauQYh4mqbLBdvsh|?w&U)A4e$n}b3o|1e;sZrTJ-#H_Yp zZBgV^m#(GwTrp_!kj}}UMo<`UTGi(RV@hAB9hNzD6~__=N5fN_v|7c4l=j0==Sue? znxW;{E|eMDg}K@oyw#SyuPL|jyZ-jEwOA2629(9=7e*8-QEN(tiVg8s4edHowScIq ze9OS9XHq-ab`I{Oecoly70QOT_7D|jXSE3d*Rk?iXPrD(c}`Cem7`b#N=c)PmrA$% z?OFNW3X1D3J{S`_S+Zz})RY;@KfA?MBYLeImNTAvpPH{YKxGASJq(#*9I+C=*{1O6=f_DR)nxFE^2(~ ze3|fnF?!tEr>_y~&Wnac5(#Oe)Mcz6PO>w%*Cm~Do+<2gN?EW83%SM}%%V4?gkR$} z?{@R;6(h#$AU#^C%JtNc+OSgv)u>V=rfg-pw#bfY5@qOLb&9xBM_o>ms6}$FUAs1(w>fu_$E!x_oR3d)v4s>_Kl&w+gWby`=o2 zg$?*09rx;3I`*J%VLt1l_}37gc6fmr*`U_^9lxDf4`+DsEmP zv$}xt_0)r=?127vT5oZ)Ke{dKKqq|%-6j)DX5iPmI=|`&I@JK zcju8D8J?U+SbndON3xZBcw`Y8I4Viqfcqco%Dj8l{y6Q&tpnkE*nXWHz-EbkuS*X?Uc$L56>FQWrMpG*^Roj(Uo7+RiZn<3>ihP`4eb^DmiMMTMVPu3`|WRaT?96wQw=Ns_6K*D zUHJ&WJC*?+sWQMH)!mZzI(4t^3sn{Oq+Axa#!38|ueNc9FS$q1QSgdU;&2;$IYd&{ zx1=k{KTWpHQyHc6eetWa!I&y8Zl@39<#K+50@>H;J+1_g*V$vX&7RZC6qRd|`_+@G zYZWq^-yy1xN)7nA)&-rZtP2A9^f^WQZP_Vsr`>4xbs)h(tyh{_+ipsCzUwcwlT&9c zf#gpbn;n+;=3%lF2%w5YMCvzY3@uVCsKDX1nH^@k0|5BBbl2zSm9I95GE+)70H2I!R{!F$H*@hAQ-;BQuPFjsD+)Uw((yal3rl6}5;nC=5`1jj*3CW_kzIr)H7rlSJA4x|bEFEMDbU zTX)qx-fcf}T0kAKgCOZMV5!2xY#Ox1q@d95#Ap}quMgbQgd*9h9gS!|Dch>PC_)H< z9bruWVp~f2(hgqL3mxy7rE#x`WNA)wM#~1E0~!T&uiz4*);cj~!WMJ>2K+Jk@QsY~ z$H|Xmll{$Y46Y{yUM1yd`>-Ca_I;18Fxv^Qzvu$$wba#j9=HITX zz~Hvx<0GUSqukK^;EtQ`kVVU-JUc8`u%AIm!wTtlad-h&C|(-RZK6bzqAUK+)=zmn({i` zJk7W2)agW@ubvj>8Cn5o+LUv<}_iSF4d z=vf-T7;o9#=#R-{vpyYUo8)o%oEDA0G8lKNv%rN?K{xBcrF~|y*<|<1`qSE?eq#7+ z@_eXs3jk4vr=G76jZPM*MT}J@0_8)Fq(-D%a_KdC=VA)#t}-L5Bvd&=1^>g#>0~vT zBG;)vbbFRYWUQ>Nr|H%-U1P(IzqRE>b>m?fgSX@pyI;fMull`B7TeM7=i3vUNU?cd zi^O!~bQ>QTI2R_-rU&Ga)9_F}tn+SSWpHY|%}{u!e~vzaw1?N<%FSCtaskR-(dZ8C zK`VIIgI2=Ap)o-)9IRvm3m{odwhwx&aVr1^$@}F3J)%GpU)PhxW}@o13WbkOn>Z*{ zW?Q9LXTzFrz+mnsK6N%+3N73XtM|T7`=NvmSzfjf8Q$ellAtf?sCNirDRJVFfq-*ig7aI(*jgA=fE6$oMr#~~x` zev({#PzjfuWeikjGVKm2_{ykoX6RYp{X%o2FWGeX@d^jnJu%k})ud3OPVKCQhKRp# zw+Xp9DvhGW69HI;UNRb7SoOd}TAD3R9*LWC02f&A1vF~HZNp{46N2WkXd*2lBW$BW zD;0;EE7&`^00T4XqUJg~5^Lo1k5s>Rl2VuDd#B#Up=^Le8x_xsMtCsJ?wi1D-EJ?yI#(dTd1Ia z7K4v~JRBd-7U(Y-t!q?+u*g&R75`4S)#e$cPqj-h4uPlc}Sw}h;X&+yiigUMf0dpOuZovczGHXSLIMWO?1PKGt~eV4aLuu4qm7T zf&%LqLdo)BHVBaIwB;adfcGH%@)DLI-W z4SFK>EzY16;%Jxt=rx1+R_9%jlnSP0u*&Q?BElS9U3O5ixT57RjvCZDYjea{}I z!o;umQSv$64g;5`LRFkJ$#Gp?r-G2(U!U9*uWQx#oc!Mf4;zg$m6Zy6^KANa2*6GW9OcQ^ zQ9lBbK7g?Wr-Ma{32ne5IbE)PYX3g#G0QTxRFnq1T!jEARaJf9m>ViihDq>G>skP@ z1i>2x!CT?}Ckn0~nqQDY1hX^&m^RstDz6+oTsAX~!>jTW=kV?sT)Oj*gI0=A6qWu^ z3RA=Z-oXRQImgyx@hsd*b^hYO;vi$I8Np-=>AkbUmiZwY@~P^mDxZ}NRIgQdT@--@ zm3(2|4C8mmFuoL~r86YI^UfN(!kb%s$;}*&+ir_zw!LTRV|WCGwTGCmiJQ8zkco}N zLc#&$69*|roO8W#d&nei3lffOFo!rWp78ew(bd?+NRH7idBZ`dm>)(!wKL%>BZ8dFhnW@1?rOq7g1UYmaCe8ww zI7d5Tp?%Y$*e3HueUsBVRz9W2qe|qpQUz0BqW8yWYQq;WWq~`HA zMa((z7Rw(30>1b8BkE>|-dUXc+X3DGH5l~2jf^dLQ6{q)JSyCO0p0{I21;X7+4bQp zRIp+%#QL#9ZH8aoH|8TDJf*JsGVOOzs`eU_D9@EECaLK}@DTE-R zQ-iEax(koet$6uqvE}ziohR>FtDOax>+$Sai9q`=G`eeif`1y(ew9wW$H}+mJ)iai z=L%NYtM>D=L^RkQhap@`eAS=Zmmutmss9Ag$eA6BZ=t>a+?A;V zC4Q#r`zbpQ_ESrwSrN8(-ozG% zT!OGKv2_<&Y~Ag8vVao*!Mf{ouJG2b$0}b|_fv)3yuMG>Eas_l!Q1Xs%_K?h?No8d zB?$XERkO&aYS!>n<(k^8(?P*oyB?Gr((VU^<%d?XODiYnt%{k7JhmNTrfAghIRAl^ z=yh7oZ1o+jOO@AYZtFJ;XZdGw@CZzRY)x4Xz@Mq-A6lLSb{KIJjR0*#%aOeD~&D{GE?NIV%9#w!8Oh6Pj|sb zaF4Y1k<HpHbRE6~(EAA4;}L(tJg zp|TkbZwBj2sus{@v6C+*}~tK5F0E7E)wc3K+&RBx5}rvEM9&h{I0 zP@54e=AZ}LZD;v%_p!5GJm$o-TR1FyJInL#?HP!CW@FuOP&?~_x5b&|ukT}LyQJx7 z<5p?GgE!Zc2y~i__t2|v2;TauH3haiuN% zns0x3E>KU~8!PKd?XC;n7Msgo-^bo|Ngc|@GANOwOJ$F)J#Nhl=hz+{MQ&w>?ue)b-|glVw7KwW3vO=dS9lH|#x5_{K%KN7URy zRlq0e+4{JetW#>nia;6B@*LICbZ=IjZH3QRb!a?lw-e&S-cP zVW+_!CYuN90f65x)4!fjlw`X!f3rd9NrmN48J@)tSL);}SSO8{Y$w!X7B5nR9`%V{ z4>d1TEv)g?DIOL;sRz*(n$8u?>4Pp`I2A^hBdA5F0Z`2v$jBdM%faZhhQZN7ekvh9 zL5JbLKr84IH~J+}9oWGp719^WS&9~_n{=c;2D3Xk&b)AUbtIGAUf zEouXsmW*gsi5=odeMd+w9X7%~hT>&Uss3*cLs!ta{1KxwfQa^5t=BuKpH68GKYfSk z*FT+)sKc|B&u>1vIs)F=28TsGoVz-xr#U^wI|e197WDQ~@Gvw9co_XhM1YrgdYaC+ zx<<9bBd$9n=%TPO-$EH)+p{s>ppCHCdb$9Pdg7bD^?8+no%p72wy64)BO_2Nx0H#g zq~+o3Q>DIT#{_vvpE@3#lk$KaH_V#^EMp zmkePWQ%_3S5&IgH%xSoV)^GdoDdUvo!wZWAfSR5t$%G+S0or0NbB*j-qM?sn@icf+ zdw`mxqRG?wYMCvl4*zyN$+jD9C&o~2hQns9tvm))qn&AieLpX0Gl)+c*hOLU?A93G z3$at6{hD@MEl=;W5*4uU>D(pR43%NCt!fydyF!rORLLmgAW^cm0cp8P*AsL;#7UW_ zTi&qaqnf1!l7}%{Yqdm)qk+~TIamC8;z7yn3)K^ovC|I=#-CrvX67wm&~Z={;HqW9 zjg_i^t)Z1IcoRGa{`%%uv>yCL9Qqf+71YX*y7EF$oo}+g8E4sp02znjvm}x&O!Ih$Ze-;P zcL?+k$Qit{4C_J+?H7`_YdD~g@NyW?_`Qx1G-DB#_ZUZT_YQfZC+(Uif`|aXuv~TE zGPyQwetBx0izHt>PiZiqK!WyGR(YURJboh-b|mKKxEUACAYF(4B z{8UrF>_P$DDMv?xsfolsg`p74p0C0LHgEz<9jlw0BS<73r-dvgw^ zfRYmolkq1W4lxZUmxk=P1JQ_d_Lr6-EW|uA8L)ejBIH-5`v%m19k&E1;~)Psn`Mi8 zC9bTmuxjV;D8*5i@Py>2B;^-|rj!)cf)5F$#HSd7xQ9bd-fd2$AlT+J#IS$m)RLX< z!r9`=U3D4Z>Smb7HYn?+DsL@CAHIha$z9sTp?CtqSAukoXB|Z4vt?*4Tn9REE75q$ z4hrIO?!}0L)kg0W`FFM2myfS>5U@To@mC?>8T*Jh_s&{qL|A}_sscBHl~CxXzjCdL zyoG-KG$vqRkF7MnQ+bJ2y z0ahv7$@IsMP+24-loT{KJwH9K~qU4CE+(XH=ZwSrVWR7gQHN?wi zJDMqH;^~PPK1mmJiCcM8->rcdwLO{2ANf4ERG*}a&C@zXI1yKbXV(S#bdpH~)Mh`1 z*XG2Xo~SAH_K~}D3uo#k+DzwgyUq$nNj|M1kOg-_s=zuqUFo8Jo*_ji7e*`H`IjGp zGpP#(+Vh(gXoAOxUV?yAFA?V&!7!v?;G8H+Hf-66n$F;SW~cv8m^A5DL;mRdm`#E$WM$ge*CzPZc>3eR!CwO(=%`s5qJ3&3>oz zd*l0^ceN^$?*Or06%ZkQ4u z)(~T5eAeUe0y()N#lc9C?`tEEMq|qE<(P(yk>7Hq1?6+J1xg!ul8h+oNG}i0)d{;> zr++20r^i)1LUMCzyHmZ(BR>->AJ}g|ZC*+Pzn7B=nXW}o8qE-OzVXve9_=#I%gZCI zc%Sm9n9jeIeVq4V-l{w@=eB>nusZc%U{v=RYB=E#=OwmuKOv;&mzWAnmn&$RjCzeR zLrOh6H#DZP&hUJtO_%-zu`<*?y3{6D>Ri$Ttl zw=nFo$znS~!YL+vTrQTI?PQzc(UauO4HVzWqxyaD_Ui-wn0;POCv*1sn-A%8Hce?v z!_dfrfkK_l=+48_Bg_EK%!cDnQ+n|8yV(|3ufHcx!cgMy2yk+o6+oRq_i_PsmJ~qo zsg-lyI9nA6gG;k7)Ht(hCXGLRLrmJzO_}uoeP^8aoRyi+8Q(wku&B~Cyv?rZQ!9Y3 zaJkIX-g<|`+>tU~$8&?!8nHdOVPfhFMy1#O`wyYJ;nkb}_&Yp0?3?|%&d{-Ch_tEG1V2J;os2{9EF6GA+t{=!}@F1QlwAcY@ zvB*&|LcJHkxNjPnL)m5|fxY&?ix~%J7GH@h_5q?fNtz*Z2pG-+2>gn>Fbc)g zw1=FWirgd)$<-wwDb0^5oSUgPiZlgmA$GhB2W>XK6d%*hqLiRY6A|8cBy@@1xvlnB zoDNebn(i_3v(>Z$Z&m>6tX%62pXPK0XpXj z&}g7kIiyNcp~=tskFQitWOy?Ch;7Cmf%zj>1p7Gp{D=?L7Q;=DEi^BMYk8LbNVX%D z@H(t!YV^N|(-&9-dG@AD^oyEoHrf4xG+BL$$RVuCu{7AD3(p>1uss@S`V79s&jQEHwZtuFXM;Jh+Y#d;K} zzVdZ~pPD6qGGpdXCoyO4G6>dnQ|? z;mnHH?ZL3gZik<_u8y7QDbf7MnDP?3un@!d9_f|42Zk_+xDVu*3O+QeJ#<7X@>Pq` zN2@uFK|%ymYXkv!5_(}SOYhhb;gRHm2xvq^9p`+NBr`*N?o#dd~&piw9v<9k32dC#=4c>aQ}aLzH&VvO+}>n67yp zDk*gj$wlAtoC-g(Vy?@XR@846g$?Ob^tmDH1Xi$jbc;dO1S4I1U(KUP4SD?~7%lzz zsShTzGJ$w{^S2>|52?C}`jn*&@bs}G+U!9y4a!fEK1D-QvX={~#!aPgmo$%?SpkP| zU|yzrqa|R)FmPm>)$)Ppgue(GO#M`Ev_QfCL}%yt&xcPRw=M>71IejPlfLk=C?{Ix z$Ha9kU@ltMkyhgzSE!o%W$X^Y5)XIOu$o~oQ(uqv%6KB_fq95TH#i|;emx~8v1>1` zWjGW5twMW5ygzQq$wSx~N`C3ROH7Mm?PVZg9|3@uEjek?&`qBV z*ITpN5|GqPP4&3d+N(LF>$*jaJ=r^$c@<|45rR>DwcgOOJ#oGVe$S80Wj!K62~ZUm zR*@{9VerP3szb9SlOzU|vSeaHa0|thcHdParSBr1#Xbr@dz|uXW^kaQWwi?q%bv@x z#VF!w*4S8tT6~s5vt5<+rS{)S8hymRE`IMXC;J7Uis$)N6In9GkdI$jK!?c40H7HBNE! zgvT;pT0AjW_AH(#Yx_yaRqwR>YxD8sF~8qO{Vs~!jW)P?_3@95Y|CdzSruYmcIHh=(UPO zZ1V-|d06a0Ba-Wq5vr{2hR1SgzZa}onlHp_70a5XFSNaGcg>4ZoCs9!y}vHsH&54A zSASh*!1x4Z2aVAKz~0-&GNc?$+t?iDYTJQw%hrbN$0KfRwQX!a1hlhl9CghOv-0I? z7U%TQ-ky$#6bDLLTH5h|aoXT7QnQ%t4Z?-oI`7ts$&6>z>lQ)E$Tf>Q!%1Bnzxf~t*gwk%?zc{ zdswVTzN@7n`Y8y%MI^=_xotBD^AHJ^#3WT~mBzDR>PsUQi(FyjU!0*bCoW;3&X=T! zHhv7K;QLZr+F~S{a?*BU!U=jWiSrX7tSD;UkR~dOSndL6|o9!m#J{j@BIV; zlD!0|?d%xoEY(V8Hd*I*=m>?;sA9%J5fWo1s`jNL_h{dQ#6dEI^!fB;`mkJX(gX!A zQPaPAZMmmg%lr~vz3do)y}rd zO|MD`xXNtJ&M7bjtom<=fUkyjK&8Cf117@^CYK%q0PE1l@<-X0;Y*T(s!1%0q?@L* zbSE1H7mO}pQ@xt42ORsr*Q0r~qo_k;S%(EWFCAKGm0z(uKGem!lYe|GuiSa=;}Y&G zppb@l-j3)NpYN8E37SM>3qoV`c=^h@M?@8xs~HVvZ@7tvBEwYg1E&l#dU-Pub_mPi ze7K1@k(*z+*w#y~GO;G3`2-v=qf7ahY6u(;UMpPwh)Dc!%yLTyf}=Gx%PWSvVn&wB z1}{w7d3QCA58;F#ka29$Dtvtnx05rNor=7l(`}E!B`iGDJGCYfs}h9<;2@<hEjE zQ>ro;C+n1|exad-zb_lU&inr=Xof9kHCQDRqv$`I6EVq6B9=|MHLq4)rJl-&uBoIWOpiwS(19EeJ-VG^GSDN zE8a-*{Xtx%BpH+nX35DHH9UU^{*~ z9NNt(SQ!aE>k?LJMUN_0@SM%p>Rpm zL-1VZ8`H{RaS)J~=;wHFPOkhSDMhhQ@9i6-f#eBiqulSpw0@D&_Tnk@{=;`rnf_jN z>aeg`$=my|;5+XZRAivkRla*WpFE-4&Dt$M3tkO-kFJcuSIp*ywYSsjdF=pYt+qC& zJfd(6AcYttFtgj~YQ@f+TyJ|*Zr{mZt<;3r31O`!2cZ;6VJ0TGKdrcleQTN`^xJ(#|~cH0bhBX{Fx@{k7>H1j(Exa(_}qk%K-jrFkrP$ zjd36tubkq3U}?{1~g&<|JJS|j%%JpP%Uq0uQ?E=csMiM$0Dx0({NY4F=}6c1FH{=^WfZ0a$c+ z)n`}Z->jBXCaKdjM@DUVzI;u-U~H!1$oCkP)x5`WSgZxptF`2?7bqA41~x4@Rkq7y zGMk~N?xuKb^a->`dYJGvkRzj>SD(I1ezBDQL0^!#CW5_ApVKvZC1M9k)K4O@6U**0 z72lNMHPCUb8vrt_)U%B)6H`)l1U}qt>K)i|w+RKB+j8ZOQ?g?-EKS*gCWVr=UqW$M zj7G!lf*ElwiU$cOGFJ#GqeL+#P&ui@JQ3Vg=eTGKMRE)1hCZH)E_RsQFG%tjaGu1| z7f0h68WrX3wX1Vek8KL)qOC3%s_9Bnc$l2L&A$rGoeL49r2wI+7$|v2=PT5>pqZh3 zPl3RyXh6V{ikaZ2B*d#L0QMX)f3w|gt%lc3uq?8*Ot` z3hSFbd>#&|t-bC=#aa_vil~f4!~1){m~D30%URWQS)+tXm(E;n4lWrouu6rjx$3e_&g79Rf~{a zCZn4x%?F1>L;MWy7c<#nldiWWfVkDvSlPh9mSfX_9sE;V$Q}ajX0BkU?m&Q$!r0?# zvhw3#k6IRh&WX1!Plw35Jv1GPK=%xhSI@1ozPkOP?_DH6mj$7v6G_`~^s1$BH4}UGFsoY2Ns%~BNeS?XYjSm z8xsmOT5weLrC#O%%eDPJ?0WnHw_{L-cSx&7WkeDl)>w_ow0M8eQ|5xJjtr$O@K{Yi zW^t}dgdtc!4W-a=#ScHMkQ0ayd)ZID4qZcc(_+ zp^LsYxvn;%GQXPZKR545OS!uFw^5rlbA49XE?*Oe`n!M0@70oDfS|gSNp2 zJ#gYHZwZ|06+n(5y1p-`i_qcJi!ov3ygcCYIpa( zr@r2*>OQC6AWQ@j5CbwMg8vXPq9K9~2q+}NWI`qp(KrMHaVC?P7;r@YKrxDwh`(p8 z{jt_w&)T*3u3gnt-F0vGz2{V|z4ltqdcL1$J&zWxpu2m@o2!UYMacpho6ujgv8b4W z_O_^4s5i4MIWw?sW$Du>1XDqrFM=TQHR&GBWM;&#wjE4M*XR+%8b>l$B#or%^ibk5 z1>NGu36Cp)S{xnK<1KnF@b>thgRJAx^aH{qS5sVdqC$i0MsolUP6eSJL_mr&Od_Qr z=ABYm>UD%OLZNFlhfd_jRRZw{PdkVJDKFa|jBU400D7kil!`WJ&s2+!L__8bHX=?; z3ju_QGpQ_sV&_fCmYQ3v%FSzhe9d*KC3H59hg)DYioA&{VP*&z##@!x9&+3iBjn#6J z$Qzv!SiER1%dRXh$sjb+m#Kul*Iz7xn+fuyL_s*G6-<`?%^dkeB1G7qEW20oFwJ|P6ur>IQr2=PWiMHd(MXb(mvaF*W&#oy+vrF~d!**;`332;hSk^fB3Ljp z%cL;T+Q_TqN;UTmkUL@~xvo@mkHc0@Ldg9k6`RwpW9Ppj!p#oxl)fTT$>qI*K(}Ty zSH3$fbdb6`2T*5Rqz7Ua+?8fv=cI&spTO9i-$R0T!QkF0Z0_a)}p z%HeHwPiDaqBqQjWbZ13DIUg~MiNR_YSdE*$U~S;HBCiar_iikTE+o^T(y0U_PDZ=1 zL9tJ9{hVG;^-y)3(JpA2V>P{58=J9X9aFf%6(cp}B4Z|EX&H4gM#62px=zL@R5cQC zC5k0e`eM|vc`Vq$nF@0#@tAX^JK$+vi1HHVo}^w^x=;aVk7kA64Syfa#K_}1^ zUa|7+0%CPfuOVP5ypRI9J~wid()Of`9593mWBeRnY9^)@YRjmFXF)i+{DQ9%tFws@ zzzVc!*Rwjw1^zZ5^EkbBE!oRrr3JcSOxpl8Y_~if4?BG+iCrkS6i!~p`1|D>-mC)no^yCZ>7T|pyGx}aJ9b*D1399y;s#MSi> zv23$RQifUnb*GS74iwS2fo2!4=I;g7eTLwqAsfxZBwncNV^km20M``2_7DZ{`>6e# zsX0+@mY}x=X>EN}2S6pmey|jk+?~cSxDEpuvIGt2Pv_AY3omHqVrjWKY$&Kr433&d;dTGN&jeY2vUa_1iVp z03$moeVEvMiE#q>my9{;tKsHm7h<=2v>Yg_D$h?UDxX(l_RwMCAN>|pUb6=XkBg#~r=i#$Ppu{<0s8YW-P2BA|JTb&he10j$Cg@x3MJ*Ku5Q?+peX78Xoyx&tZ@;y8m_9A_wius-@w(Vg*0 zrxS>OuIQ^44nLqw#uG7cJe^D-@|HfHt9i{HuJiQA;?GX&@)my?o=#C$TKT;B%D+eL z5t_DxIlNkbe$(l_^D=%gs`dVZH=T?gh$~`;H=Za@chNiY&fg9I&Q~*B)e74X#4u`) zgn%6_MuF?~;nEt&+`@Q39nDPr^kK^BO^7!o?wMZ{I0|_n=6?yk;UR4Zz;8TU#me(j z7TBwyV8y})hlLFT3k6C8ObrbQF|JZ1+}!>_kQI>P#;Gu&FclQNa}rc52yQhFNTw8w zDgXpc(Tc+;z>7~kx~T|mwMh^`G2(tyP$d8?4l&&{L8g(bk^y3^*j~am3(tRlwZ_B4 zdr^g|AP(p|Lf_ni`Csvi;Hq8*W>DFy42Hy6gg@Q7rKHg6+62e!;jF!C1YedVJ-EJ|7S7d~MU{wYWeE3sC{ zHw9+}^h;r80><8HaiqAXcM7dBq9^0HQg|Xx50=cQLdwC=RtQX4OV{gli!TS&N{#m? zlNLUQdlkzhTPS#tI&65FS!03f`U>wZb#VOR!1F+Ei3a{@@k2?yhm}@k$j~@x!JiF; z6bxQ7Rul%0eht|7ame5U6gDSzZ!&k*0>WdV-E$?7ctbil$JQmU2hS;G@uU^7JXk6a zA-Gc5E=V~3DU5s(X@yo=8R8=CFb|{D&R#Ie-Yda;iXZey-Ts+}Rv+=t8bau5ln_{P zKq?P!R466NctKWAGzC5nx`#F-bWkH7r!<9*74iGZxg>Q{I>CXj9&*(7(0&iNj8M4PX|?> z^vC4e4BeC$UeEr6QM}NO&*L3xEuX(eG9b!9BaKgJBT@Ld)-r)ZHH3s((N$5E*@C%Xdyu>fnR}%UA2fTO^ zcu@>35H|}Z_tW=&hrCi9Stg&Bt)_dyP_K5T)0WOu#8E#XBGWMeQ-DBk{2>~@Nux{$Gu41iCMNrE+ zat3a8255l;o+|GyZTe8@Ww<_(=Vd_e!Tq}+*H|1a#$M$byTK#F8mMs+zq@D3FU+uD zsBJw{c)--C2tS^JFsTL@&lD2UiDl{$HO`+yhE9t&q=^08&b$N5*RD;w+UKUPJ{&N$Pc*$KGh zT<+eibw3um2dmuiB`Ri=!B6U|IAGhtPultUiJfj34R$34S5S4^3n@TSs@*fHo+c6LpchoTx3CDssOpI z?D$1CWrEWs(yD}ZZz|3vTx&-4B1Al6F6(p?7zwD)A1J+7WO$0u-4z9S*7iv-q+Ash zrPfT>8TM_>RA^z})_lCynknx#3apsiqzeDlwm4XWI3pTK)2;k_TO`YIR&1C&Dwsi9 z%#xv~^u0<9!xCayZB1dd{ho4>2G?llLLB`M%_ZgCxC^*ZqYZ>S4K*PmIUu7{RC;hc z8;PW-73@Ecs}=5OSu$5y8Z8KJUjfZ9q1(0^y0synrxK8GI3wvESF+_@D#~nGvBW$Q z3fCgyx1ijlaxbncRw~RhEEe51k35w6!;mgD6h|_82XRL2ZX4as(&0$}5ci+(4RZ5+ z)Z3-DDdu~`^aY@wOAo>VU-jQbY4h;~zw{!*wRKNvi%sv6;M+M8!TJgS+`wfI|6_sx zk_f&Exq=C@rXIZhM{g#Hd!U*{ha!mpt0S^@Pap3R-3JIjM=^kw zoz+0em|x#Z$@ya>k5GmckwQF1BhrE@CsO4^a3L}&eaK46B1UT@Q8`&b$|!2uX%G4n zp%{0i^~J|fwyfMihBV73LO{!qf8HRXLo0GT;yAcd)6=+c3sJJ zD5)bvfsJapfQi&FbK4ljwcH)nD|5kl?-fh94<#<~n&o~eZb%md9mD06Xl~sKHMfu; zp0(yiygR}eUg(mwjpEc$-&SzaHT8pBN~fyxY-Q=>j%NbBdAu*WhAicgFj#Fi%4%)U z4&rszsR9+aASwm&xIn6&#D#2CKGs^1jcq_Lr5TeVfl1d;jro_Pc2)8vJEk8K&W9$T zMMwTpc*bOMyv;nUWK$N5E21-4wji;rgqqKJX?IoS7j>Z`lh}wzWvTq^F1N4#qstz- zxbZ(q{@*S2--X%`+Egoq*1Ez!kJQTaal2F^Oei@Uc5(wI&sqJ?1?Rn2|6Nk%h_yTg z_1~p!P@eZ@)c=qm*5v>>{aU-}xFy~lVGOVG?Xdd4g{51RoWkn=&f6-gw!D?a;_wpF zfKzGCU>77wWm`3-impusUd09!ZWPA`6bGfsjj7cyOzrlm7*-^0vVsqLVOZ^ueVBuY zw5-u==$Ez$V-~Wbp&!u&sFWQT?`-?FUsKwC_#a*R-Ywg&CSAsqEw<~nU#$b}R%o~z zw*3H;=WP4sg7ewP{xG4jJqgdR~Dbiea9P zVs0yqvW7byRaL`4*ic+3jxybiYSoBub5$0VlEP+Hgx6kNS_ywhUs&Eib)~3m-TKw0 z4!{uYK9X8k;%csWQ})TCMVLck&_Ru(gWaqSVxQDEsxa0#v%3``2mqD{F&7{f$b3;s%aG*B|kb zQd}`dxiw`DwL<7ST&U3J%2CcuZnhOwm`^AF#~^%%i+HJ}#c(pe3me+ljwHrD26{tZ zfu30#5VhNiR}zH;V}CYPX%}HSr9NoJ%TYYKOvO-WNK{xOM)LCGPb4c6V`GT-kayp4EeeTPmj0TI<~?oYt~WW91!Sg@@o3`M?9V zYltXyl-(1ggr$x>?`Q9i!zn5qT=JjdV|MDO$Vr}r^b@;WFdt&!As5Q3q1}6_Slcfx zMe8u;Roo)Hw6(q8JGMINGkhldz2mq*YvRE#<9$eDqfyeh8uw2LSIK@KPZT6rPyR#D zQaRZ*LzLCG_l`mYPyp?}QzKwJ?5Y`2NdkeG zP^mA!fkcvu9P#_O^)fobQ#tj=lfSX1tKfB(${RyG2{Uc9ZH!e#Ut8=d?fAAp*pb3f zBty)mSd>TwO7C!EM{Mp>rlwqpibGL$N}0qIAZXIC7CMuf*J8^>SzEnBEHg;@Ntc%* zq*ovh_1+&)Sz$C`|1|34zB%oWmM(liZ%GYAi|O3$W3yGmZ+kuNM^zE==TLF1R#SA2 ziez_SG&P9rImicf-r)B${GL>q-f_T3%IDZ6wYFL65V9oxFGYS@{ z6)(Gthf-Qusn#=AxWhKk-xUx?3yP5?sin*k>CQJY!_p~?WqdlA=jr1|McVwcfYnBP zg-?>{6t|NKFyWj*JMb*Dk`NS3kvoO89V<*Y9HB7bhQ@?fwzfRAgcDB9a6)mYtVRyx z4w@1ParORaG#&Iwi8Zd>gdjl%Vj`E?3P@f`peZ1!Yam$}wcjX6toCGCYwHYHqCldh zKw|Aqt#2*J@wN)&h)Kp2-Wmdg!aJsh@(0_oY^PJjcFbL9w`QVU=ptF!OF>h)g1xY} zR}@s@R4A+Cu+(HWj#5T*sz|y$3r)cEo|MYN)^T|?v)@1SqXeL0wze7``>F|)fXZrM z?6)RR0whmV6DR?Rm}E>D`skWKiS3xX*bz;@x|cG-+iH22nygAqAlhaTue^+bM9R;I zB%5M`jOg-GKzFH50_on)bX{jUqjBrFSpH(Ch%XwR#-7)^^m^a&dS7_GFUxqLcM%rS z`I;9Litkm22J3ds)HYdE7$qU1E|5aToaD`EG%qDLv~n5R7Bo^qbn+H*W9~1aiWjF& z2Je$vljKW*Yr%vnq;gT-8f7V~T_WR4bxqpsprgZmRPB^CRp0>U7 z2k-Jb=SeCJeO!Pz?yoxcCs*s@y>v+zL;d3-sDJGB-g&)q4x1EzSn|BI=t@;PdfOQB4~@jbmmi|78zv8B^Dc36NLc}_sN1$S01pn8iL`a;Q(@$6mb_9M z87Y6v&S{cy|Gh8Pqda~-6pLC30T)ei`@c36>*YC!&C7LHE(*Xdo<}%#< zdQaW-Zq?FwRDeZUg|Xk4N`<@BSpU1)%gaN*PEFX8xe9Y^Bl{}OkBtrm;bV$|C^|Vl zA!bJ-oA7y4-b+_FS>Ydk zrf7)b>jLjE*b^HC#B@;3@F9#z^C262GTqF^ zwrPPX-zKLWxpb}Z>qd`GD3TURiT}P)tw!#N$*LtW5Mdv0&jzek{yD6Db{Xig=BJBVjLg zr9dodx$eSny8wG>1e1Q-OGTUN%>m2DT%aOjS@JN*{Xvd~8 zUfMU53jBn)S@2LQ*hobj($0W0GO%+pVA2QU#GRGul*%Z9Bw*mE`MU7u_kKr&I?mpv zZ^1iR4VObJF=r>~m6RT#1+#SxO~A6H?0IrFLlk&rT$)Tl<_vi8vU+_r^J%L}n+E@; z8G&{E8Kf7BvyhmdtS(jpIBC&qsv>)-iQnH!PUk_B6ei{=|CBIKmx0)e(N@w%pjy=wwf>Y-t}Qkbs)X!s&+@2JMfkWl z`qpRtVBR&fW#J+Q#X?O!!T;jRVE0M_3-U6CLMAj++$F93 zbxyCG&c*rbtet`aoFn8SjJ%t>{(RV_i~*;86O25O=r=@PpOKGz4RS)@AG@qCa^&9{ z-FG~RSxBA$52t_{JYaB3B0zw!EQ0#y9`#}#o!_71;{pi+m_tvSDwGoVl%Q7(cna&) z;m5Mx#K^W`OT_CGK9;UaV0)C|pLrjR-Gi}jk3(j2ZR{wU3uE=>P0gNrfw@n1RB^s- zs7Xd{3sgHTKT22urLv&nbfvD6os_A5A)_gilN8P%`eqUSfP8O}Lj=Bqu1Y-zfiZ`; zPtm;L=K^ENMe>4J;UWx_SFn;J1UwmyPH{YF&K7C8#>q7B&g4swQ3BKDnhY66WA)22 zS;Oi(w_8}2$zJm=E9cp0kc_-(IHlFB_%V>jWb@krNA%tzs=Ofr3{YG>; z<6`hq;|fVcG|6Ew5-F=->=i9!eO6x4Le^)Z`4vRyF;-rMn8?4wOyr9gbcXS> zLSE%xh#l1Hufi56>2#vG3y7dGm}WWx+Zj{a*vGgnn)i0FkQ+#%)}86a?S&oLz2x(G zH?Qz{egCA%7dZNR_~!5PJLn~gug2FDIg#>Snic0=#qHon+_F66ygOD7Ilf5?7m&PE zR^X7$AS)ASe4b?yERi&@zHOF>C{6qEEnHC~q9E;q4ubV}S;W2kSS0@lJKbSQ$IVJe zc7ftZOHZuCYzwhdv3Zy`@+jDTbun4RXT>nVM+S{Vn!J!v#6ymlY~mYFE4ZMJD2Fvy zZg(jYuJ}tzm}kXbQo{5q{*n?Zc}irn3Pcpq*V4wAxC?K1wvg>d z*X5OKNBD$d!8DTYuAU)!+YQsm=(85%h{tESE-PaN6 z<__+w)i01B9!?-gSDDOy;}!O5Lb?SCV5|)`o2y;$E&&YXa%j$Ix{~Rj+%6TsP$shj zBW5lMNH$*NA6GR)b}d~#MoveeYe>;-V$bo+1zS99w3i?q?X%i&XmjR7VJ-9X0&)c}PjCEqc_b9*t&uS8Ax|P->inF5_4|P)6u~pLI;?q~jW_JK- z$iHaGDQC+wQEDTv)MJLiV~mOP0a8hFGN4CNDAottlT#>4fJEp`vDn{3g`(u1RnBPQ z`7UcvB{sar3PrTrB3@fPTMkri3oyam8HeBK&++9*%F{d@*YI0kk*|rom*@x9wcTM^ zIBPR+@M-&Dxt)aCenjk2dhS*Nc*Z+RxPW5SH>IJ0wrrIS1cMDV+eE z3-Lq~u*Gc(6P}tmCtP-sb{rpqfGTaW1ar)vXBLKGq%+FS*rdu(EEtnF?`r#*W}Dcw zWKDVZtRxexm1{OakSX>~VO#V5gNR3p^HP#8w78|fHv|OB1u4;#Y(&YbvW8_3{p;}w zD^pL$vXXLkrVB2jDLS*c55}UbXlP!PiKmH2 zV_r{<{(2`Q&^8yifoR>{K1ygG1|Qpr^VpcMN6JWuKDk{mN*IGdAKXAIqYs(bZr4o; zxQjNqR@0IP2+gz_Z6AiqTtHCJ-KQl1w~PEE(iO{l8EqQ)M(}lW%dMWa%CB_;_!=#y zfWzc!)VVqtJ;oKIyh?FZhQK52bnDtbSD7SUJ1Lq5cqTr~v|QC5 zK-pF-KxF(Rwyfx@S*73jiy~x!+e)z+3jn~B?KyP6q>MRxI7#AGc1KzM2!N!6z8DSjsHb(nuW3Mv^MlKOTYa36^Kx)hPl{6ULq__l>KRFk9;6H^+!K@m&tc)(U+^UN_O z$upOAUo+n<*G2C~%se|(t~wk<(_#F_q4vX+spe3snp3C$P+;9@uE<)0GcETId=(qN zotw9+p6~-#Rif~KE&GNo(lL@3Syac!`Xv~dm|iwl!MCvv3gt`wsLWjDHP!SNttLfa z=eA`i2(9sSc!$;yk%%y~Nw+=pulskSrRRSf__Jj&op{T6f3g_$m(w}&pn6NEmV7l^ z&d+(zi_;1wOMgD;j}~4q8KBnxwZ%uHjvV@#BO8+|Z`^;l@;$6(91ObWj^j~T=jqkg z{$Sa0Y74B=j~fd>^ssI`r%n?|P+ZS_zhl`%J6L%AQGm9N{3cudtH4|KZ!BM{ukk&^Vs`)OUK-5^%$nyVF~ z4dMw4FrNZkSjJ#F!FA}#l3hxWy^j>YZEtc`;X5F0L-96_ba9Ke8`k5DN5pWcEf=HX z`SdE7fXZgw^LsS=FgQdTH1FN^7q@VR2+gmJY_$H!TTZ(ePA(Fy`ak_u_OrpaVn7GI zLyj=abLY2ib2k0p&%NojxBM{mZsyYu%Uf?Y9R-7juBiE}IH}Xi11~xjHQj^B_4M3W zsQhl&U-reCoYrDF6v#uvqaC0jKtuc7|8BG|C3<#-fNHpj8p$K_otNB1+&m)0CZ~mi zN;oX#_E}HN#D}jbyX|X1YH8@~#^G6UYo87;-h^>@D+$!iL7+DBPY5&`Aqh@H%n&a@ zt1hACOXpKCN%!c;IXVh?^KK#r@D^ojMM)nJ6#n$76xBJl=o(v;525FVHxc?ua36pp z1=r|Q$GGa8&C3YV$jY}^5<^ItR)u%wouYAK_+4M z_y|7+8&&ze>nY4$&Y))*Q0bgvjPdxb|IkApC}?kIsHWvOU-Ug=Zrb4XTlJMy}h_@g){`NHTL#o0P<(+OLOWsUzyR1vR*xT$$4eRdB-n^Ln! z3(4f!CHLNjKb|cgSU&PRu+oLj++PL*PXNLTkp;S9D1|l*|2pY;2Ke9Fj``_vMyEOM ze~iFBnBKuKkv8k50e{sWyf*-EgL|611W{@-i6nN%X)nNO`U`K=Uo68-Dca2QpeUy? zh9NeZ_J`hepSV`%xk)-e061+zV8bZ=%G99+BgrLNt#n8vk!uFGJ_q>@y}KD~YS_Yw zecX{UJNHaZ1O6*!0tr7=Db7o;ywF$l=PaS0Pd_id6jCz;+YlOvvY72~K+SwmR1OJc z*Y)5`dG9O!RM8DJVVR^$t)Z7ByHDvQD2xtH3Y%!?!}&FR@TBg84>8b%k3VtaiWMb) zUkG#eU!*vu7bI@|?ZFI!oL<9YY!RXZJ^#9?wcA<1o+<2X1C;ZL034IuYO$sTHN zUU6L8#vqg)#(KwSGOfW%RXBER#SBjm=`))UwhdVg_i=$kLH!+rsK)m6m9D|f5?eCG z&4kmH;T0;ok9ozWt-UtsC5zyDDx*-5e`5e}L3qnsc*(tyKe>Uc+@H^bH^C~8CqkM&L}A*&Aa zUQJJ`?nM#q*&y8VJX|3f%w{o`CL`DL(BL-7C2|N5+kuud^2hL5L0*wfta`m%i2QH? z)0sb~0Jn}1>n-ABj99sY@bS%rGz)Kub44)h)$H;}W9yq;g?OjB>&@{d$9qXJQPN>9 zDP;ggE4Y9r(TGE15UVtw1^Vbz4Ty%0@-NFzpNT!=@277zfJ_4I?!Yk69nR&m&N-FftA-?oqC zm8wyx>co!epVoD53}B}AIT$63K@*}ITSJ5LZcK%5<4O}`b*wZEAyTb&-^Mw*b2RiB z%|h^TF!t>zk?lc$)*l2+#AnkdQGXP(1H$mCzwo6iDe+M#*nf`rn~-Xo1^}v5C8)T# zod**&{IFH!0wH(dSxdms1qSMhg(B%IlWFFAJ2Cjz6h3jP%ewPKLTt@>B)7Qzi`UnR23d`4G?;`1ktQ}YuDyqSW(XaveM!6EEG@NDvK*F_fTY7-I8g0zX47JuE7 z^R;XO(@MODji{ccyMm!DqPv8R(e zi$E?+!BDws7f~G&v}e5vQ$UNa38G&C0$mtRF@NHlud6X%t65!T>7+%LLts+U!S{ek zY&hw(2=FBKbC_0HYHE???B7{h6XzrsEA3G&0=0}otSrs7@OmmuhAr~;J;b5PQf8k# zoHmhITLfr!QnLmq6&YUG;Zl5Qn&FY9$mBDBwxD^jw$e^;Pc77|C|h!I8%~iee)K6R zPIX<^N~C?OI`bY`keNYDRbfXS)|*{g>?V_P*TxHBM?7Zg?%F%BV)o-E_-<}rR@t(_ zh#E)c3P{TvJ_Z%r$Y=Yh5#48&?ERT9xAY$iUpCBu^1`;g941jp3%^&J zf?ei7k-9+CwCzOg;!EYyxz8zNzUgf4;!8!A@x6M?kInCMC5FGz6GO}kBRcQeh6kY2 z;j@Q3YWn!=4X2NnI({TYgnV!(6cXID;I0N(wecNA*tE*0(%6xA?lOc{q`g3lxW3irmH7Z{$irHA|z=z{-mU zuXZVO<1w!%6o)wo$-GR^1^vXA7=(JOB-SdzVjC*BxjI~R2oF4ONKL)3R~STVU5F_D zwn(wr%HGz$Tv2#~YhS@A{Ko1oY7~BT(d!&VULbE0>qS$Mw4QP4W8K-;xEfOA)nOW; zMZOd9t7aFuVJp2rQevsQd(EWyW*+Y{O7XQqc&&G3rSL1)y4X_uZPE3Y5?4%$yo_|h z{mT19t2WEKp%jzjB++(k9OvPPrilb@~Zn1y-bq$1m;(&?JUWjBB?{}l!hH&L5chPrmg^er! zCYbE>5^Gd_Gr14SRGXgo;iaZ0*kZv!`C;k+6rckYl|(}i4XQ}jdl!1yiu1~S73(}C zhEc5t>07Sd_-akX)A134@1oDqJMev{w!A}?)Tjtu_e+!>z{j%UIWh+>sbf-DjNl#m zB`A(IV`6mGT5n>rA1L3^t8@0Y_6_f7Fv1J}OHGGlTu-7kiI?1zGdK?e$Kw z#NA$G%e^Do8$?aB8IK_JhsgX2*e)3hPhfJHk2Kb-ssGNTTjmcX#o@I$*tbQieB-Zt4^I?$RTJ zKI{Tqr-E@>N5j-SXkuoY=SZ0ddD_%QlW6y}?VUe(m)|+>=r*@i%0)4-sAi9v<|5^s zU<{69Hc_+AY3CD9TWQkNj{|g4wYb*PKAt${q)qo?I?B@q4$pyYn#`x`X)@0elooyG z07|8abh@-)HCoYgM7PyWsw}0X2*j#XUG~-J*OZbKK9MLcZ47HpuDt8J$$;8!-TU)} zpQxATuY;cJo!@t$;0g`21qHO%{VEwDY)WIwS~R8TMX_o$WiW{agay7+!ugHdY4 zJ^?C}g>)onoFl;I%U7M37oFF-ex3%8TgQg4Lk_XlGGS)Qzc0_EH@HY+)nJj|pBQ2g>!v_+Qern0EjZlBA{-X9)ia$Z+i7L zy40^R3p_=6XNjrPSc`3r-6UAYSmFJc^hmEd>d-AoUbXF9!BYU9LG8rk@msS5 zthZ-$8(io-OAdZIf~h+WvR%sQcWsi=i@JmSplHS&QmGYx@W7!nsW@U z6$&(TJ{io(YnJc`a63E0Bd)25rG(ieGi38SAyoy_BxfDwTcE9Ce|a}o^%@|KUdc#zC<(Q#E!I2>NF}*tHMOeG;mLwKqfSx@0<@JjFPOK?sJ)!*wT;l zs^A%~*MTwGI=@84d$0p>QQXq@DG{;4T-o()9d!WSv+QPfF2MOVJCnOk)m(n6p7wWaq02= z;R`6L)48xf&R$%+>4{sBirjbuxsk(|cge7vz^oeuizSK1$a0Wxf=l4?Vdi_E^%t*w zA#cwe$D`(1(n^tcG1KIEDlAdK21SE%w1+_-Ubr6=jUij!pSq-Ph#A5sJ!%vUj^WR! ztFVo&xa|dG-_>0(S|W!z9_}r`!S5EG3sRC=#@D>?ZwKgR>_L~7F=aB_=o5bMsY^*J z%9f|t?r7Q{CJ@cuv7?LYO-Ogl5f8c>8hBy$^)$y}0|BsocY{)EFx zoBqdHKbD|2Q4Bdd?#D_0$tdO|WM8(FwgSawy7YvFOs%~AA@Fq7kT^>y4ZRdN+ydYL z`10yCksf4BpX|N{3D_6_Gw12eG^M?&uhcQ7ig#wh$8ZN4L*lzn@hst64RtvVF$ny} ztpHM@1rfBc_W9s;o=E#|^^g5Y{|cUr;JW*zOiTd|1+3zu@6WzH7990r=hj{|+-crT z^4ke%Cy1s2of{-5)hFS)di?b7`ZejiaBWJjFOvP?f;~hnlh#wbGBXQJv$KSPL&U80 zX94+{nzvwuQ1?4^vKFkC5`$@W63eLpsktBdad98Brv>(Qh2QF|cjobD1?&#pZwK6l zFHTX^kBRt&D9r^!Df|x{B9#!+Cg-a6-hYrlL`RDQx{==;#aVQ8LsO@_rPK$)>x3|F zS*j(qR5}w5B&)-4HOC{^5egotl|{mwJnsO-FR1%?7y>g%xe?}_Ur)o}(h78|G!?{k zhwx8$*YoMv3n;1$8-MUVa+e=;ScCDpgXwtIpZg@G79^&G%9L_q?z0(_gduTJ`C?T~ zKs~7XeAHz|KL(#8Kc9$9rSC~-d4mVFFRjfu6aOQ57m4$9dnexM`90M}7_Z=j&lUOV zgQSv%K&D(7298b87@n$hlWMA1pjp4TTp@JIgG#BZloqZ4H$)DuU>pP7fJkyfNFbP8 zPetB|aS&3o4`XI+kDQ*tVoA0 z&qa1aIC%@@f^$8kHP4Fu++iRPp6Q&^Si1WbU5$uVCw|n>BV2SAt`KY;3?r2ikw+d9#j5FT>B(8*BMq}TazeIJP+=pXvnG4K7V*8 zGUPmpOueL(#W(<>2d4<@WQt3-r{AnH$5A{QKgkiWuIxBiU7VxrZx%A8!gj?ywRIdF zPiFt6h5f!lK=F3bWyR-+1fdUxwtv<`HUwWIEAhcRANJt1M5?^B%tM&Vt zz3DYw16)2tZu%2(?X7da_&T|iXC>lR7~>Q*b);uf-bAl2=eR`ONA9V-SSQm@ZOgVz zBa)NF#k{YjN7V4DMK||^2UWbJc|8t3HtP@1%yQB%bb^|qJDoMU)5+Qh#G2^-<)yfO z8D5gI9g#gIv$yo;AUho?_F;AxAV!tFT+el8fj_`ar-hHZ%1B_J-wl=?U2=83hA2?4`UvXGZ1Iy?1+tv!--a2XUCY$%lD= z=FNRva`Y#-IemvdphJJqf53&4s4)2%YQL*rP~|h3r%#ui&%JlO(%EnON^7n=_eBoi zlrEXzCfnHe#AS~~CY#P{Wo3!85=8x}O4OPlv~kCr3q^VRP%KKRC|%>WNuMCgbwidb zRFCf34EKJlYAKS5$%=zHxz>@_MZH4lY1AtP2>aBym|&|EAZO6-5K{2~^4n1ff?RR@ zRHcB=(v<=XXP{}7Nr|pTYP-G9y(`(Alqxdna_0alH2`b@)ia__CuGueb1bZCQy>kc zr6l`6ak`tt7eJm92uZ4W*L}Kc8YU$?pWV0SJ{0PEU&Wnb7o{RztX=0`&5B*MYK!E> zu7w$Ih!;~S3NNk`^XyLEZG4L|rY_%IbY6L%y+9$GcW2&3hyD~IgA~q9S;?ep70+o! zN;fOy1xe`QN>^=M^@lMeOP?oZiy%9%@x;#G8U4i6bqh3R1yG z)g5xVMUlFa)7@3=a;Y+>0YhBlJI*4EwOA8DJJAlL9fDFp_&Aa@5pj01*03hKuNI0g zI4Gh<>8q?ATvv>Fq+A(CX|R_R&p;%zQuNQYi0Ggz$0I@PV3T34QK%+B&k!Oc#yW*( z>eyIfL9(>1l(|&$3gyr~RZrd7{@bx9r$r@4Z|1ncEP}E9$+CMze)X{Y!SgYXP!&w1 zLOi?s6ix%?hzJ+zj@3|iP}B@n^`#&^x(~uCIi}DT)g|(n19A)eO;+2+i`p z>qIoglK&Mf$@X3n6iG~9yci#;Zzl9_crqgZrSygYuAy~LzFf>?8*Hre4c1A27v`v} z2g0mWQPfqa?8Zt{ZNCV5C+|eg0|Cg=5L|(qIHOnz)eJs%YLN<2W@=4Wblm7%P!951 zG(y9o1z*zNsu|DRQ?DlWr=AaUBQ`IRab!}Oj6f~YCQ)?k0zY(OCt8=rQIfVMNkM-| zqXqjz$w|>tMEZwGiGtEQA-d|zkxYOh%O~_NoG^l~I3c~#-$-3?;7ySo@991;h&y6B z6gAdKy)9Xipmf?|SdC8ZvBYvUKs?%V4eHS&vXQ|X-KtHUDABSGcYLw}3&b{Z zgQ~fh5UMzPfdn)FhaMel2^=F!ABg58O@ z_|U|?I4WQ&doz_O{7D$J#lz0Lu^8z<=tfN-vkRpJS281HFH^9 z)C#@qc*)&H+)gaiatw;577zjqxX57knuxs1#$(j+W88nZ@=?KKIt~V1U0!q?SLE;X z2j9I5&?-f+21~;%gBd$6Wws*o#|!&D49F!e{oGV7qkQ3KC+0;-k1$7M6hkxxXf#4S zzI1e3m=0Q@!>8MFGsWS)IXv)`}+yj$0^Vy>F?$+sFpKIDj+>P<~9eJ1T6sUhZzxQU-Q80Mu!27Gvg0Lt<-qs;;kw$*W*sRSmKt=zk3PRYe>Hu5PfpAgP?J4ydP}(h zs&9CHub(fw_A8ned1&BM7I({6=2GLMD*Mt7Ea7i>N4Pr7y8sksg;yZ_h~BspxOBu8 zlq1T!L30Z+XQuh#Wzbjvz?~m>41B7Yd=-!qhE999n*`sz1DOWCezJu0l(|?ib69DaGZtXfn0W#_5uFXDA0}uOej8x!h~e{3 zLvG-nFJ%+Q&t9X`E{Zim>c`KrMCOw1Dgc997Pwu4oBMg+>QJjK=>l8bEHOFIU_sCH zam_W^mJWAzp^}ctb2gez-`}O-%|Amc6q{RU$IK6oFM@W^gh=cRkt&q9fTH0RF(kUJiB&H9?={rX z+S6H{77hsUwCL^@y^CF1f^6UO)fS-qV1jo7JWoENEs1|Fh9DKPpk{Rnj#{{Z6CYG+ zym*5eHov;kqe@rI4NAkcL>?)z6Lk9(B?DIBO-z)7>FgmH3HdzO2$&?*HfXZ2Wh9NFk_foc8Nxf1VK``W7AB}hGcSl;522ATfQ_VVv`ZU-i6oe9h@9x z{YC=3Y8-qOcymX?PsbvO@MN=SUeoS~!tF>7%1a)IKezht3^W&MSptS9a5lk@7p9QNKd+fV@SRGsNDe0~8YdMwVU^N$;e_L? zi_KE$Q}MDB5u-Mn>Jz>QFXKd+Dd|kC5W)z;$$Vw~=~QM@-m?3q!7Oxhq=tr_mm}R= z4{WkZ^eLL$UU?t<{(I_WEgLsyJf8Mf?nZ4+CAlG~rlSVvc4L{&QCRl7kA~Qgw5!S@ zh%o7EJ>2#>En!d?LBy?ugn|5W*wAI^Hi1to=|f&95nYH7zM+(|i?d75Ig$ydWUz#8 zz~TY%y#sE^uvNI#6k)#GIz z!ZYBAijL9}eMZc~QO^QkpaPoufvl{C3Drpn>OCje!6{d=^r(Qez;~)WszF?Qx)SIm zN|0W>wH6`OF5F}nGS$`!9Vkjl5othJJCvDN(s24w{%iL3;>P)_K_+mfa**2EvDREw zA@wvRvnl{!plGTfAp96|MlCt`_;s$dx;RA&I40oa(AHkaQPvO8{D3GMNy+Q_nE;ly zi*_2Sg%|u@NIyva3x4K1*StQbt^g}sanbJt>n(WEZ&z2W-;Wbl=WwE>Z~SRB64<#_ zp;YGLpZ12t3z~RPuz2Ag(BHo5=3l~6Ry`asN(m*67JxaLPh7hdmU>S0USvL(>@mmX zH)c^GW#G0+=(?m*60myLxcP)zC3H8gf}2m_dR1UN-g-?O(!E3|7Ny;OO2v^B%r1cy z>eDJXD#Q)-6^`OOdpwBe!Dw~WDqtRO)d`-&Yhyf;JhsRiUlh|;_mUaY*21soNRi5q z(!PEbef^x&ch&Tfh7Vm{h>{Y(?7;0-D&hz0hptu&@Re)4K-Uj-sOJNG84Jj~+M%De zY0ZR%hiqiAU27(8SRsuW!^%)M10WvE8??qLV&25?mDUY|m&HsRS-S)X9%rJ2-aYl8 zAgGJjJaqI?eF0%;t2s}cot0w|p-2>W$Hi?g+k!~(j`UG1f)ql0q%H{qRv1BViyJ|E zvjyFD2(6;*<1b^&==G$`A|_cFm)lIqTz00SsaCsOF$ynwqG{%lP1Zg=N8IXOw<5xcZc~{q z)u#uT;3H|2ly-&xg;8Ey)=|T}QNqUa!501EFKOKQ6y<}3vZjRsFz17G$|SL&<|7&n zaw`YJU>(CCgrT)-m0bxJSX4;cZcJE0bk{X$-wS6{hWn}1zan}5_L+9=a5N%gyM zhJK@5Boqde8WG%sav8EzOBaT9jv_v)i2#O zJL^nR)=_xVBD3Q)!u;AvNPRYPDaI*hT6c&l`d@1=HGOX>ZSSdEj{)73Xp?=B(I&fx zHrd}kD3hF&-#|0%B3QdxpqF6@pQ$V%mo8ALf`U5pm2~DV0?0j7L)J z2UR<*YFnCyAN(4MUG7xT7W`?;mi3#5KW+G=+SQi6topQ}wDCsdMZhOCWf(@9zVM1n z;LWrvuV!tVYieG6eHqtxUB9{ugUFnSMmbhIGMZ4^s;}cVrg;ComvuV!e}>mQ(ve2hcX8zZ2lHpcfaiS z7jxGM7NT^j>_I%g8!UsXkuQ)%))iXJopv+^Z2*FsNuLJL6mc^C=hQ^ccG$#W$;et5IL7%X;x<^@SObjMN@i zMuKlR3`gn4kjYCP|Hzs}q-VcTVZX_S>t#*_IXcu7{Cw00>mgoUWQudzEtC>N}aK-c>AV&1;_O0*NrqR@!P)`?D2^;YBv2+R7@7 zcVk2BrB5?)X(JasQ?IdK?zHJ9&oxQe+Nxf&i=s?3GY?Ipds6Atn)cd-)r>h*yw04X z*rnGLQltv-Ki&n=JXKrFwW~mSytOBnTT?5{aiLWzr!1SWdj52}EaUm4tIMkHvcJtS zsklDX8Ss$bB`Uu9vZqh?OH?4h#}S1^Wf3}}t1dhSU#d#30e=U@=MM1C*-3MK_$NFp zp``=(bAaD2)I-l^x&TC6tX@a?L$ZPB_mIABgS+NX+uJ4SppX52UfFbhx#L{9{JPGnR!V;2Tu@7pDN=Au z2p)k@beSpywY{)F0WfOzY2-5zZz@!O;MGQ*QtQ+JP*+B#LAE#Q^9_5W9(56qcIt~V1y`59XaYaHo+6ebQqIobs%c)DY(S!CAAFxtiO2e_m_8bU*EC3@1H)k#=IXa&}j|rcgo7z`ZrT7T0qXm8D8yq(r=iLS?yH~eqESdw) zS4=9#4W?tI!#OBEVAmGT!qoCMtkC*C0uw!!Ao$3(A#*74H5)z} z^J1iRFo2p`1drs#I4j{8BW>??+{JS433U)OoPdE{fJ3dr<=_GSWECH7GAWJoAdtL8 z@I9f`@LG>kzj|2ui`)xoD8sXtqE4J%%)MnXZJND5pZ6c~F33$klJ`}$tLb!<_bm=g z>t*i2^=s%a1%}ILQf^cb4Oa7xc0muh7f#m6_k45{%m2vn*dI^l5BC8ykeq7*J%3lX z8P3NMpUB&~7VT!wA5N*$EZcU`Fg=h{Dqre-2$r`YO++#ye!KZtGq$hLWNnG|hY6ud zCS)V;Q`9bxOtbJ&(*5u}QpCwT1kpp;h}8g^P4w(y>EEpxlSxx7?{ZMIRfH@Dx4zoG zQIKIygUMoU88BWiND>fN_KJx#`B%~eyB%iHs7c4$ky_O6}Gczb@EF*49b z_SungD|b#)ophNoG|-1GGKSvioBBFqXrK?ZDDHCqI@lkwX4Qv&WQRKSU;-=r+7pNj zZ$(T%mf=Ic$nY8ry7%6P{(KTlZWb0>4i^!DmcbxcN`rF6aAG}6m|E(@!jv#5$8FL` z%jEncPzf--rM1>`ZAb;8a~4Sbun zziRzip!&Qz2yBi|14lT0kGyt&Fz^?PDfz|lx5?d@%$G5~4KFvP=d(WW%CVeJTdWna z%bZI!79}(%(#rAAVAMU;-E_#;5PR{Ol7)AauA{4=Aunr(D_=dF?oX$;Iq}7HK(8%M z5Iu=Ikq}Dei@zI2-izeB(0#!jR8Z!i;&;KTnevpxUs(^!ISLl=Tnt9I=#rdQw4pRI zxDC;q9_F9G8aNxS$;y#Fun6^S597MV0M62(H+vXEl1~TS0j_0s#*#n`_;AB1lOesw z$Q#7WZr?RC7zG%^rKESwtt9T06%Xc|jBe~?Eb8d1eZxCydtNZPpT1Yv3NeSGQ@y3R z&`vw%O*Dz6F}86E29~k}DRm6|m(sH$x&)OXc|oDh7*yerrC3Ws_WhpVU?F`Y=?i=m znF1_lgl0v)X2sbbKBhYR5n#*h?KhPiCG{{=a!#H0`NKP*L3-rs);oObDki&X1FWr~ zKM(F>_kVH@F$vEM%#Xi{kP!+MT=J*W$&J@tyo5;VL&JRwg?EIHBv{1RyVNRN3=0Q+ z(MKh8W~3n^Q%9DFkC#q&ByL|tu@>8T*=KOqogo6lSpq$s*BX*0S%0UKI)NBjg`lQL zL$ovfK;uJbFZ>BY4F1T%pqiiFU0?fi;&}QD;Tps5O`herstZ4^6B{ou^}QjM_3vXK zCC|5Sf{_>U=KkQjcfs7pDpR7wVMeW5>&owsjJKrs3d#+me7)sy61`DL5<^m~LEka< zlFX9qI{hXy9Ar$hY6z7*f<9o1e0`?+hueNg%o#3P$%mr5Z0Ajw_x-VNP$W(532}<( z;=Bt!2fnM>W%3}`neKRYx48AL`h)jMeFni(voN^^jTpVJYE3NY{^G@@=Y>I+b(Yec zU->t|L|KQry@{WEK_W!I4t^d15rT>C@`hK&aa9T-uLWX5q>&R4siPSs%Ag0rC3Te+ zyIg10^*Qv&WN9JRBacH|NjDdewe${Ct0u;3otK0saGXAX?BSO4atbb!smQVp2wy2J zXCwHf_mPqh-5B|(i^w{5&Vi7v$vf0Ckw<>U=<-q4IZ&{YRR{=qQqt8<@ zapy>e*vUsLljt@1lX^P!E}g2$P|;CG|DHK|Ms_oL1T!k0OOmd}Uln$0Exr*JYbeR- z_{te_DSC;KgtO;eidHBj*-F+cZPH{swz781Te4WVU_-8o5OmnN9rL!N9Bb@*9j2_6 zfKdf;Ikn}Ls@APP9o@M{hSYLCeF!rPG`;Ae*OGeUIuJ15F&G)xQ0)sepjWbq1|`zW zjb(>aTkb4jiX!|unIdxU;}k~zQqj#q0WRU)hRG+Ou5cNUEBb=>O+wgNRPTUNI3L##WRZYomF?X zi(AB-RKq}myo$4hNiL{2ff5zrxsPfM(Glw8@ zk<^e2>S3+NpN?aAD!2tKreu5=p($imyrpZMBfm|?hT6be4Edx16UQAx1lXkjB_bVy z4Dgx`z0)CjZK+P=4J?>ZJJw^5(E8>+LV{zR`KIBpKooO{)?VZm>_)r4SS9-eSY+U& z3I&GhAW~f@Z+eG#YA_$%jr#L&QPNPsiD@gv0!X@QZV@LvsNi;DOLQ&3x5xdDBDW;L zDoxWU$9CZ@_m;PFa3$pH-XMELScjHZtj8>v4bTP9kV&8=BmSZ=`E{XAmgHDRBIg)t z8wKY`Z3FX%Vz^L=bQN|+=MI+>nJ~(9V6&hKBMfUt32eXH=*!>T%YzIFP17i{&^!Ief*MmBmNl@u``Q3w{v-3T>`q=H~_J&mA||MAyXdICV4 zcfnmSOug-sH!oj?-gIwC*X)*6@o6B?B%zVe@{@o*UoXf1wt(j198K}E;6QFLx zT%X%u*3w{}Z87U>n{alIO>C;U+-K%|wVDa%8nfS8)7(d`-89a!#=yDmm}{=r%=zjK z6VCOfxxPAsl8qx1HyW<_Y@^1EW6rX{oLhrgOQUJRw9zu*(rBA|YaE+!X&#w)&~!{# zH68Q4O@@w|%vm;>{WcrSTAC)Tn#`HEj?DeG82oE7{H#?s@wCOzeyeG&uf@=Qi{VXe z$E3G5!${k8b02MnrrS+sT+7@?``A3AW9EF18Tv41tC=5huro)sO)6lJk>M-HCMr5d zCMX?-jXI9G4#zP`z+rafG?-@@RCE|JbC}J#4EeZ>XmS~}a~tMSyDbwHO+qn&Za&2= z!rVJ2k<63f=)+t?+g#|exkJZgK4TcC!$8Mj1ck#a&S40~VFVt3%H}p42B%$y@Lfj5 zsX69;Ys}s_;Z|d)gcn*SGiV%{CzM<8FiK8?VSY`9Vw()&H5oK(x+cyx89Zn*jHt;_ zQ{{In0tA zZbuHo&>V*`CmmMBbr?OyaTvsafMP($slE;;102rmfdOD@TlkQHlFPx9^GG=^qaDEh zW0Zl~5p$X~PGHv<6c##&vp?LY={V z6LlGH8>2fh>N(^3WW3#s^R3OWGmh&r=8nUdZ4RRnvMMXbg}IYq#JS6eb5><#R4$hh zWiEs2tdE_cOqUUFE+b4hr;p1Vhs*fUIJMYi%%U2zUyei7m{>uLc`v5|*BRDYXVi!~ z<2tD`rbeA%z;#BwuQQZUXT*D*!TLG_XD$X%X9Nr%7ce;uuVypcm=O*QhKX|i{wBkb znhZ>v4E;74GqmZN>R*#lFj#jAW2Q8jeKZ-EHW_WH#q5K1u`s4uo7qR3SqrZgo1FTX zk%q?%k2q#bc3v$u@!*(|-OdrGq8u?40v(tUo=1$RfDX)!h6Om#IE@$`K4j)} z7|x2PxyPB<7+fdZoQ#7B>Mo=IJM7{aM6;>8rYv?i$3H$~1i8yl9F%%S)pr?_5HW3& z5TPxbTm~@a*2%af;9_E+RO8lJV@MwQD)V+Ohzli_Ik*}p%4KrQ8IcHj(v8bWKFnj0xEl^EO?%^~TR_io4+H7#A*fcrK#cp}VR#g8gqX&Z%P6ESV{~!;1vQ~wOi|loS;w&C)hKC<<8p08? zALod30v|EV7QSjzGT=kz?N~iy5is1rfv(JqWW0|U$;eYSN(z@NGbbna9mZP&+ktu9 zWzZLHZEhr|u(=GTTl~^YCK0^NCey<%nYY(C-x?<);2CBRjMvE-_7A@v^Y%KY*47yb z0Gp3FP0m^4)EQ4EK4h@2&KZ4m4!!FPbF(Pwj6cO`Fg)35FuIP@;8f5CBVM5VGw?Lf z*hB+UGh<9O2H1>I!^XRHn8@&0J_o_v6PqB-%*WZpZqvjJ&LZZ*M=rzQTqfpN zV`#L-&cyTi`~_3S*O|T5nUEMq z(Y(%To^PGmZ=E?eRzGFVvcc$Atn-%f-ZU6}lB4Yg=j3I`jkEt73=KAzJO|FC=PddL zBg31FF4|=5DBihja-}9C!kUb3#^!%8yph$t88=gtu_T*}-PmMwI5z2=u`fB7M2iXH zn*!IwWm7CNNXCd7M$jxAt-DOfwxR5q` zIf>Z^mox1$JdMlSs4;;TW*ZFHxrm`@x-dF5gR4x$lsiHbLpY)|(Vj6?7}8}pAfx9q ziZWv_GT|r2ImFnMjET(9Cqtf$TF$U##-YvxaTxy;%x{yMaCQvm`DK>NaTAx}fVmS2 z7_P|X&oVk6n^3^4nG5f*m+_cT2$us}XLx0ukvyy}$6y#|uW`Zo2BVWU82yC3pvUZu z3#pjcVt!(BP}7bxi8+tl=j4*9uQO@}7w$5b!05P)I>ulmqr5XZ7ROJ_MKB1!uoy(y~=KvRuH&Dy(GBTZ6DdXE^SSTk>%{Ox5ltB%1+l)cVpPUIp_RBce}7`Ti0p5zFYXi z&!_(RJQ$aEI97gddvQeKP-aj)@QPSDe(QPGwZWEg%7#cx&EoGVnVLd8%>vmj)#9D&*AV- z?%L7hS>^f%qi~`-6mM(iA}6QyDT?=;PW!`rBEuN1osx}aTjyv{-j2>cZAID_LvO%XgqYls04-=RR-}C3w1+~=iBGoOxpB05Q5PBL7 z$fujuLfy1O&F#)mq^&C$-U<~wW@`N6H0)V(fsVRBjiXqkRn3W**g4?V3x@s$+N(?_ zd3f2%XSD^O(@cLD+gNCwhqlW40LBCZ%B@=9?y;7G4W#R(sUx%NdJB9h41N|dYu!3{pr}juZ!Ore(U&c z;J1n27Jl3KJ;tvi2BP~gmgLEHU9>^snB@-&YukY?)c=(#q3g^;H%li+e;Kq zc8)OH!kbMOo^v?WCwQup!>K-*Q+3~?)%$^3gR3XTgVl3A(Cd?tt&Is$>w*7175>{m zQ-Z;q&J%`z9E}U@j`YWowweB%(>&8-A}y3YkF?{uq5VUrl@q|7Jk zve510zmn=4Ze)Why8ce>YeJ-)XE+SKte{%|;5 zYI*sY8wbdN|$fp0274R%w#Q2b%iB;d~$Od=K<%cbM&|?5$ksPmyc<*8|;d zm2MN|e?46}(&vo?va&yV4z zKAeFPDkyUX709i_8I(GMinYVTDXe`82U?iUL?n|7X>L*`RHbPhXrK%YRHY9g%3eVV zz~-X=i2K{aP5HEX_NFc?)Jy=jV;nx+1!*A>a>!RC}KS~_J|_TG}V z_00mY+H4K%+g^|RQ7I~+vmI5)4(Ey^O4*j&q2i>sbgtD%ufw_S#<^CnCLT_9_fGae zvrkCIR;aulXmuxOwVWO-6Ca}sm7}wstA$83^?;54O;1xk42^eMznI!wmF!yA4C8=$vcUJ+s+?)M9!Sh>(S?=<=mEx7?OL4!; z(dlikuPDB6XMO8%ob9i0Ct+BM>bC;B%IOx?opVHOFua?){(MN~^F6116O6pY0M#9a z-rOJgXctIfBK%{w|F)EYk_F|*-bPYD1e+3QmU!Slp| zqKy0roqkd7Z)+iy%4!icF_o;QxS?L*+$Guh35>g|

Sparkle 引擎控制台

+ +
+
引擎状态-
+
模型-
+
最近一轮吞吐- tok/s
+
进程内存 RSS-
+
+ +
+
+ 常驻专家池 expert_slots + + 32 +
+
预估池内存:(slots × ~160MB(g64));修改后引擎后台重建,期间对话不可用
+
+ 投机宽度 k + +
+
+ 最大生成 max_tokens + +
+
+ +
+
+ + + + +""" diff --git a/mlx_streaming/server/app.py b/mlx_streaming/server/app.py new file mode 100644 index 0000000..bb7a8bb --- /dev/null +++ b/mlx_streaming/server/app.py @@ -0,0 +1,104 @@ +"""FastAPI 应用装配:OpenAI 路由 + 引擎调参 API + 极简 admin 页。""" +from __future__ import annotations + +import asyncio +import os +from contextlib import asynccontextmanager + +from fastapi import FastAPI, Request +from fastapi.responses import HTMLResponse + +from mlx_streaming.server.admin import ADMIN_HTML +from mlx_streaming.server.openai import make_openai_router +from mlx_streaming.server.state import EngineManager, rss_bytes +from mlx_streaming.tui.backend import FakeBackend + + +def create_app(args, backend=None) -> FastAPI: + """装配应用。 + + args: argparse Namespace 风格对象(至少含 model/k/max_tokens/expert_slots, + 真后端还需要 expert_dir/mtp_out/qn_config/spec_slots)。 + backend: 显式注入的后端(测试用);None 时按 SPARKLE_FAKE=1 选 FakeBackend, + 否则 MLXBackend。 + """ + if backend is None: + if os.environ.get("SPARKLE_FAKE") == "1": + backend = FakeBackend() + else: + from mlx_streaming.tui.backend import MLXBackend + + backend = MLXBackend(args) + mgr = EngineManager(args, backend) + + @asynccontextmanager + async def lifespan(app): + # 加载放线程且不等待:真模型以分钟计,不能阻塞 uvicorn 启动; + # 就绪前 loaded=False,chat 请求返回 503。 + loop = asyncio.get_running_loop() + app.state.load_future = loop.run_in_executor(None, mgr.load_initial) + yield + + app = FastAPI(title="sparkle server", lifespan=lifespan) + # 本机工具服务:放开 CORS,让浏览器能直连 /v1/models、/api/* 做探测 + from fastapi.middleware.cors import CORSMiddleware + + app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"], + ) + app.state.mgr = mgr + app.include_router(make_openai_router(mgr)) + + @app.get("/api/stats") + async def api_stats(): + return {"tok_per_s": mgr.last_tok_per_s, + "rss_bytes": rss_bytes(), + "expert_slots": mgr.expert_slots, + "loaded": mgr.loaded, + # 专家池「分不到槽被迫落 0 号槽」的累计数。>0 = 有前向拿错专家权重算过, + # 输出不可信(调小 PREFILL_CHUNK 或调大 expert_slots)。健康值恒为 0。 + "unplaced_experts": mgr.unplaced_experts(), + # 专家池命中/读盘指标:hit_rate 低就是慢的直接原因(每 miss 一个要读 ~3.3MB) + "pool": mgr.pool_metrics(), + "model": str(mgr.args.model)} + + @app.get("/api/engine/config") + async def get_engine_config(): + return {"expert_slots": mgr.expert_slots, + "k": mgr.k, + "max_tokens": mgr.max_tokens} + + @app.post("/api/engine/config") + async def set_engine_config(req: Request): + """有多少改多少:k/max_tokens 即时生效;expert_slots 变化触发后台重建。""" + body = await req.json() + if "k" in body: + mgr.k = int(body["k"]) + if "max_tokens" in body: + mgr.max_tokens = int(body["max_tokens"]) + reloading = False + if "expert_slots" in body: + slots = int(body["expert_slots"]) + if slots != mgr.expert_slots: + mgr.expert_slots = slots + mgr.reloading = True # 即刻对 chat 可见(503),不等后台任务起跑 + reloading = True + + async def _bg_rebuild(): + async with mgr.lock: # 等当前生成结束再换引擎 + await asyncio.get_running_loop().run_in_executor( + None, mgr.rebuild) + + asyncio.create_task(_bg_rebuild()) + return {"ok": True, "reloading": reloading, + "expert_slots": mgr.expert_slots, + "k": mgr.k, "max_tokens": mgr.max_tokens} + + @app.get("/admin", response_class=HTMLResponse) + async def admin_page(): + return ADMIN_HTML + + return app diff --git a/mlx_streaming/server/openai.py b/mlx_streaming/server/openai.py new file mode 100644 index 0000000..b2eb3ea --- /dev/null +++ b/mlx_streaming/server/openai.py @@ -0,0 +1,338 @@ +"""OpenAI 兼容端点:/v1/models 与 /v1/chat/completions(SSE 流式 + 普通 JSON)。 + +生成是阻塞调用,放线程池执行;流式用 asyncio.Queue 把后端回调桥接成 SSE 生成器。 +客户端断开时通过 threading.Event 让 backend 的 on_text 返回 True 中断生成。 +""" +from __future__ import annotations + +import asyncio +import json +import os +import re +import threading +import time +import uuid + +from fastapi import APIRouter, Request +from fastapi.responses import JSONResponse, StreamingResponse + +from mlx_streaming import config +from mlx_streaming.server.state import EngineManager + + +def _sse(obj) -> str: + return f"data: {json.dumps(obj, ensure_ascii=False)}\n\n" + + +def _delta_of(prev: str, cur: str) -> str: + """后端上报的是累计全文,这里取出本次新增。 + + 按真实公共前缀切,而不是 `cur[len(prev):]`:后者假设 prev 一定是 cur 的前缀,一旦后端 + 改写了已发出的尾部(历史上流式解码到半个汉字会先发 U+FFFD、下一步再换成真字符), + 长度相同就切出空串,那个字符永久丢失。此处宁可重发一小段,也不丢字。 + """ + if cur.startswith(prev): + return cur[len(prev):] + n = 0 + for a, b in zip(prev, cur): + if a != b: + break + n += 1 + return cur[n:] + + +_MASK_RE = re.compile(r"\[(NAME|ID|TEAM|STATION|CAR|TRAIN):(emp_[a-z0-9]+)\]", re.I) + + +def _mask_notice(content: str) -> str: + """工具结果里带脱敏占位符时,就地追加一句原样透传的约束。 + + 实测(2026-08-02):工具返回 `[TEAM:emp_ypsl004]`,模型答"乘务三组"——把占位符替换成 + 了自己编的名字,用户拿到错数据。系统提示里已有同样的规则,但它在 4000 token 之前, + 而占位符就在眼前;贴着数据再说一次显著更硬,且模型无关。 + 只在真的检出占位符时加,不给普通工具结果增加 token。 + """ + if not content or not _MASK_RE.search(content): + return content + return (f"{content}\n\n[脱敏提示] 上面形如 [TEAM:emp_xxx]、[NAME:emp_xxx]、[ID:emp_xxx] 的是" + "脱敏占位符,界面会自动还原成真实值。引用时**必须连方括号整体原样复制**," + "严禁替换成「乘务一组」这类具体名称(你看不到真实值,写具体名称一定是编的)," + "也不要拆掉方括号或向用户解释占位符。") + + +def _normalize_messages(raw) -> list[dict]: + """OpenAI messages → chat template 可用消息。 + + 关键:assistant 的 tool_calls 与 role=tool 的消息**原样透传**——Qwen chat template + 会把它们渲染成模型训练时学过的格式( 与 包裹)。 + 此前把工具返回改写成普通 user 文本,模型不认为是权威工具结果,会无视真实数据 + 自行编造报告(2026-07 实测复现)。只做 content 分段列表的拍平。 + """ + out = [] + for m in raw: + role = m.get("role", "user") + content = m.get("content") + if isinstance(content, list): # OpenAI 分段 content:[{"type":"text","text":...}] + content = "".join( + p.get("text", "") for p in content + if isinstance(p, dict) and p.get("type") == "text") + if role == "tool": + # 原生透传:模板渲染为 ,模型按训练格式 ground 到工具数据 + item = {"role": "tool", "content": _mask_notice(content or "")} + if m.get("tool_call_id"): + item["tool_call_id"] = m["tool_call_id"] + if m.get("name"): + item["name"] = m["name"] + out.append(item) + continue + if role == "assistant" and m.get("tool_calls"): + # 原生透传:模板渲染为 。arguments 必须是 JSON 字符串(OpenAI 规范) + tcs = [] + for tc in m["tool_calls"]: + fn = tc.get("function", {}) + args = fn.get("arguments") + if args is not None and not isinstance(args, str): + args = json.dumps(args, ensure_ascii=False) + tcs.append({"id": tc.get("id"), "type": "function", + "function": {"name": fn.get("name"), "arguments": args}}) + out.append({"role": "assistant", "content": content or None, + "tool_calls": tcs}) + continue + out.append({"role": role, "content": content or ""}) + return out + + +def _tool_name(t) -> str: + if not isinstance(t, dict): + return "" + fn = t.get("function") + if isinstance(fn, dict) and fn.get("name"): + return str(fn["name"]) + return str(t.get("name") or "") + + +_tools_filter_seen: set = set() + + +def _filter_tools(tools): + """按 config.tools_allow() 白名单裁剪工具,全被裁掉时返回 None(本轮不暴露工具)。 + + 见 config.tools_allow 的说明:这不只是省 token,主要是把 prompt 头部钉稳,让前缀快照 + 能命中——调用方的工具发现是动态的,工具集一变头部就变,每个新会话都要全量 prefill。 + 首次遇到某个「保留/丢弃」组合时打一行日志,便于确认线上到底发来了什么。 + """ + allow = config.tools_allow() + if not allow or not tools: + return tools + allow_set = set(allow) + kept, dropped = [], [] + for t in tools: + (kept if _tool_name(t) in allow_set else dropped).append(t) + if dropped: + sig = (tuple(_tool_name(t) for t in kept), tuple(_tool_name(t) for t in dropped)) + if sig not in _tools_filter_seen: + _tools_filter_seen.add(sig) + print(f"[TOOLS_ALLOW] 保留 {len(kept)} 个 {list(sig[0])}," + f"丢弃 {len(dropped)} 个 {list(sig[1])}。" + f"目的是让 prompt 头部固定、前缀快照命中;设 TOOLS_ALLOW= 可关闭。", + flush=True) + return kept or None + + +def make_openai_router(mgr: EngineManager) -> APIRouter: + router = APIRouter() + + def _model_id() -> str: + return os.path.basename(str(mgr.args.model).rstrip("/")) or "sparkle" + + def _auth(req: Request): + """Bearer 任意非空 token 即放行;设了 SPARKLE_API_KEY 则必须匹配。""" + auth = req.headers.get("authorization", "") + tok = auth[7:].strip() if auth.lower().startswith("bearer ") else "" + want = os.environ.get("SPARKLE_API_KEY", "") + ok = (tok == want) if want else bool(tok) + if ok: + return None + return JSONResponse(status_code=401, content={ + "error": {"message": "unauthorized", "type": "authentication_error"}}) + + def _unavailable(): + msg = "engine reloading" if mgr.reloading else "engine not loaded" + return JSONResponse(status_code=503, content={"error": msg}) + + def _prompt_tokens(messages, tools=None) -> int: + """prompt token 数:真后端用 tokenizer 精确编码;fake 退化为字符数估算。 + + 必须带上 tools:工具定义是由 chat template 渲染进 prompt 的,实测 4 个工具就占 + 1349 token(占总 prompt 三分之一)。漏掉它上报的 usage 会显著偏小,而调用方 + (中台)按 usage 判断要不要压缩上下文,少算就会算漏。 + """ + tok = getattr(mgr.backend, "_tok", None) + if tok is not None: + try: + from mlx_streaming.cli import _encode_chat + + return len(_encode_chat(tok, messages, tools=tools)) + except Exception: # noqa: BLE001 估算失败不阻塞生成 + pass + return sum(len(m.get("content") or "") for m in messages) + + def _usage(prompt_tokens: int, completion_tokens: int) -> dict: + return {"prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens} + + def _chunk(model_id, cmpl_id, created, delta=None, role=False, + finish=None, usage=None): + if usage is not None: # OpenAI 风格:usage chunk 的 choices 为空数组 + return {"id": cmpl_id, "object": "chat.completion.chunk", + "created": created, "model": model_id, + "choices": [], "usage": usage} + d = {} + if role: + d["role"] = "assistant" + if delta: + d["content"] = delta + return {"id": cmpl_id, "object": "chat.completion.chunk", + "created": created, "model": model_id, + "choices": [{"index": 0, "delta": d, "finish_reason": finish}]} + + @router.get("/v1/models") + async def list_models(req: Request): + err = _auth(req) + if err is not None: + return err + return {"object": "list", "data": [{ + "id": _model_id(), "object": "model", + "created": int(time.time()), "owned_by": "sparkle"}]} + + @router.post("/v1/chat/completions") + async def chat_completions(req: Request): + err = _auth(req) + if err is not None: + return err + if not mgr.loaded or mgr.reloading: + return _unavailable() + body = await req.json() + # chat_template_kwargs / enable_thinking:接受并忽略。 + # tools:渲染进 chat template(Qwen 模板把工具定义注入提示词),模型据此输出 + # 文本,客户端兜底解析——server 侧不做原生 tool parser。 + raw_tools = body.get("tools") or None + tools = _filter_tools(raw_tools) + # 「模型没调工具就编数据」有两种成因:调用方压根没发工具,或发了而模型无视。 + # 两者在 server 侧看不出区别(都只是一次普通请求),故每轮记一行收到的工具名。 + print(f"[REQ] 收到工具 {len(raw_tools or [])} 个 " + f"{[_tool_name(t) for t in (raw_tools or [])]} → 渲染 " + f"{len(tools or [])} 个 {[_tool_name(t) for t in (tools or [])]}", flush=True) + # 采样策略(关键):模型 generation_config 要求 do_sample(temp 0.7/top_p 0.8/top_k 20), + # 贪心会诱发幻觉。默认走采样模式(质量优先,非投机);仅显式 temperature=0 走贪心+MTP。 + req_temp = body.get("temperature") + sampling = None + if req_temp is None or float(req_temp) > 0: + gc = getattr(mgr.backend, "_gen_cfg", None) or {} + sampling = { + "temperature": float(req_temp) if req_temp is not None + else float(gc.get("temperature", 0.7)), + "top_p": body.get("top_p") or gc.get("top_p", 0.8), + "top_k": body.get("top_k") or gc.get("top_k", 20), + } + messages = _normalize_messages(body.get("messages") or []) + stream = bool(body.get("stream")) + include_usage = bool((body.get("stream_options") or {}).get("include_usage")) + req_max = body.get("max_tokens") or body.get("max_completion_tokens") + model_id = _model_id() + cmpl_id = "chatcmpl-" + uuid.uuid4().hex[:24] + created = int(time.time()) + loop = asyncio.get_running_loop() + disconnect = threading.Event() # 客户端断开 → on_text 返回 True 中断生成 + + async def _watch_disconnect(): + while not disconnect.is_set(): + if await req.is_disconnected(): + disconnect.set() + return + await asyncio.sleep(0.2) + + if not stream: + async with mgr.lock: # 引擎单对话:并发请求排队等锁 + if not mgr.loaded or mgr.reloading: # 等锁期间可能开始重建 + return _unavailable() + mgr.apply_gen_params(req_max) + prompt_tokens = _prompt_tokens(messages, tools) + watcher = asyncio.create_task(_watch_disconnect()) # 客户端断开也中断,防僵尸生成堵锁 + try: + res = await loop.run_in_executor( + None, lambda: mgr.backend.generate( + messages, lambda _f, _n: disconnect.is_set(), + tools=tools, sampling=sampling)) + except Exception as e: # noqa: BLE001 + return JSONResponse(status_code=500, content={"error": str(e)}) + finally: + disconnect.set() + watcher.cancel() + mgr.last_tok_per_s = res.tok_per_s + return {"id": cmpl_id, "object": "chat.completion", + "created": created, "model": model_id, + "choices": [{"index": 0, "finish_reason": "stop", + "message": {"role": "assistant", + "content": res.text}}], + "usage": _usage(prompt_tokens, res.n_tokens)} + + async def event_stream(): + async with mgr.lock: + if not mgr.loaded or mgr.reloading: + yield _sse({"error": "engine reloading"}) + yield "data: [DONE]\n\n" + return + mgr.apply_gen_params(req_max) + prompt_tokens = _prompt_tokens(messages, tools) + q = asyncio.Queue() + + def on_text(full, _n): + if disconnect.is_set(): + return True + loop.call_soon_threadsafe(q.put_nowait, full) + return False + + def _run(): # 阻塞生成,放线程池;结束/异常都经由 queue 通知 + try: + res = mgr.backend.generate(messages, on_text, tools=tools, + sampling=sampling) + loop.call_soon_threadsafe(q.put_nowait, ("__done__", res)) + except Exception as e: # noqa: BLE001 + loop.call_soon_threadsafe(q.put_nowait, ("__error__", e)) + + watcher = asyncio.create_task(_watch_disconnect()) + loop.run_in_executor(None, _run) + prev = "" + result = None + yield _sse(_chunk(model_id, cmpl_id, created, role=True)) + try: + while True: + item = await q.get() + if isinstance(item, tuple): + if item[0] == "__done__": + result = item[1] + break + raise item[1] + delta = _delta_of(prev, item) + prev = item + if delta: + yield _sse(_chunk(model_id, cmpl_id, created, + delta=delta)) + finally: + disconnect.set() # 客户端断开/生成结束都确保后端被中断 + watcher.cancel() + if result is not None: + mgr.last_tok_per_s = result.tok_per_s + yield _sse(_chunk(model_id, cmpl_id, created, finish="stop")) + if include_usage: + n = result.n_tokens if result is not None else 0 + yield _sse(_chunk(model_id, cmpl_id, created, + usage=_usage(prompt_tokens, n))) + yield "data: [DONE]\n\n" + + return StreamingResponse(event_stream(), + media_type="text/event-stream") + + return router diff --git a/mlx_streaming/server/state.py b/mlx_streaming/server/state.py new file mode 100644 index 0000000..288386a --- /dev/null +++ b/mlx_streaming/server/state.py @@ -0,0 +1,101 @@ +"""引擎单例管理:加载状态、生成串行锁、运行期调参与后台重建。""" +from __future__ import annotations + +import asyncio +import resource +import sys + +from mlx_streaming.tui.backend import FakeBackend + + +def rss_bytes() -> int: + """当前进程常驻内存峰值(字节)。macOS 的 ru_maxrss 直接是字节;Linux 是 KB。""" + ru = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + return int(ru if sys.platform == "darwin" else ru * 1024) + + +class EngineManager: + """持有一个 ChatBackend 单例;引擎是单对话的,同一时刻只允许一轮生成。 + + asyncio.Lock 做排队:第二个并发请求等锁而非直接失败。k / max_tokens / expert_slots + 是运行期可调参数,由 /api/engine/config 修改;k 与 max_tokens 即时生效(作为后续 + OpenAI 请求的缺省值),expert_slots 变化触发后台重建。 + """ + + def __init__(self, args, backend): + self.args = args # argparse Namespace 风格,重建 MLXBackend 时复用 + self.backend = backend + self.loaded = False + self.reloading = False + self.load_error: str | None = None + self.lock = asyncio.Lock() + self.last_tok_per_s = 0.0 + self.k = getattr(args, "k", 3) + self.max_tokens = getattr(args, "max_tokens", 4096) + self.expert_slots = getattr(args, "expert_slots", 32) + + def load_initial(self, on_status=lambda m: None) -> None: + """启动加载(放线程里跑,真模型以分钟计);就绪前 loaded=False。""" + try: + self.backend.load(on_status) + self.loaded = True + except Exception as e: # noqa: BLE001 加载失败不拖垮 server,状态暴露在 /api/stats + self.load_error = f"{type(e).__name__}: {e}" + + def rebuild(self, on_status=lambda m: None) -> None: + """后台重建引擎(fake 模式只换实例;真后端按更新后的 args 重新装配 + 预热)。""" + self.reloading = True + self.loaded = False + try: + self.args.expert_slots = self.expert_slots + old = self.backend + if isinstance(old, FakeBackend): + self.backend = FakeBackend(reply=old.reply) + else: + from mlx_streaming.tui.backend import MLXBackend + + self.backend = MLXBackend(self.args) + self.backend.load(on_status) + self.loaded = True + except Exception as e: # noqa: BLE001 + self.load_error = f"{type(e).__name__}: {e}" + finally: + self.reloading = False + + def _resident_pool(self): + """拿常驻专家池(fake 后端 / 非双源路径下为 None)。""" + store = getattr(getattr(self.backend, "_model", None), "_expert_store", None) + return getattr(store, "_resident", None) + + def unplaced_experts(self) -> int: + """专家池累计「分不到槽落 0 号槽」数(见 ResidentExpertPool.unplaced)。 + + 取不到(fake 后端 / 非双源路径)返回 0。这是数值正确性的看门指标:非 0 就说明 + 某些前向是拿错专家权重算的,回答不可信。 + """ + return int(getattr(self._resident_pool(), "unplaced", 0) or 0) + + def pool_metrics(self) -> dict: + """专家池命中/读盘指标：定位「慢」到底是池不够大还是走了慢路径。 + + decode 每步要取 层数×top_k 个专家，miss 一个就得读一份专家权重(8bit ≈3.3MB)， + 所以 hit_rate 直接决定速度。gpu_fallback 记的是本该零 host 往返的快路径退回读盘的次数。 + """ + rp = self._resident_pool() + if rp is None: + return {} + hits, misses = int(getattr(rp, "hits", 0)), int(getattr(rp, "misses", 0)) + return {"hits": hits, "misses": misses, + "hit_rate": round(hits / (hits + misses), 4) if hits + misses else 0.0, + "gpu_fastpath": int(getattr(rp, "gpu_fastpath", 0)), + "gpu_fallback": int(getattr(rp, "gpu_fallback", 0)), + "prefetch_hits": int(getattr(rp, "prefetch_hits", 0)), + "prefetch_loads": int(getattr(rp, "prefetch_loads", 0))} + + def apply_gen_params(self, req_max_tokens=None) -> int: + """把当前调参写入后端 args(单飞前提下安全),返回本轮有效 max_tokens。""" + eff = req_max_tokens or self.max_tokens + if hasattr(self.backend, "args"): + self.backend.args.k = self.k + self.backend.args.max_tokens = eff + return eff diff --git a/mlx_streaming/tui/__init__.py b/mlx_streaming/tui/__init__.py new file mode 100644 index 0000000..731c45d --- /dev/null +++ b/mlx_streaming/tui/__init__.py @@ -0,0 +1,9 @@ +"""sparkle 全屏 TUI 包。run_tui 延迟 import app(避免无谓加载 textual)。""" + + +def run_tui(backend, args) -> int: + """启动全屏 TUI。backend 为 ChatBackend 实现,args 为解析后的命令行参数。""" + from mlx_streaming.tui.app import SparkleApp + + SparkleApp(backend, args).run() + return 0 diff --git a/mlx_streaming/tui/app.py b/mlx_streaming/tui/app.py new file mode 100644 index 0000000..4fff697 --- /dev/null +++ b/mlx_streaming/tui/app.py @@ -0,0 +1,334 @@ +"""sparkle 全屏 TUI(Textual 实现,视觉对标 opencode)。 + +界面只依赖 ChatBackend 接口。加载与每轮生成均在 worker 线程执行, +通过 call_from_thread 把状态/流式文本安全回推到 UI 线程。 +""" +from __future__ import annotations + +import os +import time +from collections import deque +from typing import Optional + +from rich.console import Group +from rich.markdown import Markdown as RichMarkdown +from rich.text import Text +from textual import work +from textual.app import App, ComposeResult +from textual.containers import VerticalScroll +from textual.widgets import Input, Static + +from mlx_streaming.core.mem import snapshot +from mlx_streaming.tui.backend import ChatBackend, GenResult +from mlx_streaming.tui.banner import LOGO + +_ACCENT = "#2dd4bf" + + +def _fmt_gb(nbytes: int) -> str: + return f"{nbytes / 1e9:.2f} GB" + + +def _mem_suffix(peak: bool = False) -> str: + """状态栏内存后缀。显存外置流式 MoE 的核心卖点是低内存,故常驻展示活跃占用; + 结束时附带峰值。取 MLX 统一内存计数器,开销极低。取不到(0)时返回空串,不干扰状态栏。""" + snap = snapshot() + if snap.mlx_active_bytes <= 0: + return "" + s = f" · 内存 {_fmt_gb(snap.mlx_active_bytes)}" + if peak and snap.mlx_peak_bytes > 0: + s += f"(峰值 {_fmt_gb(snap.mlx_peak_bytes)})" + return s + +# 生成中状态栏用滑动窗口算「瞬时」tok/s 的时间窗(秒);越小越灵敏、越大越平滑。 +_TPS_WINDOW = 1.0 + +_HELP = ( + "可用命令:\n" + " /help 显示本帮助\n" + " /reset 清空对话历史(保留 system)\n" + " /clear 清空对话区显示\n" + " /exit 退出\n\n" + "快捷键:Enter 发送 · Esc 中断生成 · Ctrl+C 退出" +) + + +def _short_model(path: str) -> str: + # 只取模型路径末段,避免顶栏/状态栏过长 + return os.path.basename(path.rstrip("/")) or path + + +class ChatMessage(Static): + """一条对话消息。role ∈ {'user','assistant'};助手消息支持流式更新与 Markdown 收尾。""" + + def __init__(self, role: str, text: str = "", *, final: bool = True): + super().__init__(classes=f"msg {role}") + self.role = role + self.text = text + self.final = final + self._refresh_content() + + def stream(self, text: str) -> None: + # 流式过程中用纯文本渲染,避免半截 Markdown 抖动 + self.text = text + self.final = False + self._refresh_content() + + def finalize(self, text: str) -> None: + # 收尾时切换为 Markdown 渲染 + self.text = text + self.final = True + self._refresh_content() + + def _refresh_content(self) -> None: + if self.role == "user": + header = Text("› 你", style="bold") + body = Text(self.text) + else: + header = Text("⏺ sparkle", style=f"bold {_ACCENT}") + if not self.text: + body = Text("正在思考…", style="dim italic") + elif self.final: + body = RichMarkdown(self.text) + else: + body = Text(self.text) + self.update(Group(header, body)) + + +class SparkleApp(App): + CSS_PATH = "styles.tcss" + BINDINGS = [ + ("escape", "interrupt", "中断"), + ("ctrl+c", "quit", "退出"), + ] + + def __init__(self, backend: ChatBackend, args): + super().__init__() + self.backend = backend + self.args = args + self._messages: list[dict] = [] + # 保留 system 消息,/reset 时不清空这部分 + if getattr(args, "system", None): + self._messages.append({"role": "system", "content": args.system}) + self._base_len = len(self._messages) + self._busy = False + self._stop = False + self._cur: Optional[ChatMessage] = None + # 首 token 到达时刻与基准 token 数,用于结束时计算「累计解码平均」tok/s + # (排除 prefill)。_gen_t0 为 0.0 表示本轮尚未收到首 token。 + self._gen_t0 = 0.0 + self._n0 = 0 + # 生成中「瞬时」tok/s 的滑动窗口采样:每项为 (时刻, 累计 token 数)。 + self._tps_window = _TPS_WINDOW + self._samples: deque[tuple[float, int]] = deque() + + def compose(self) -> ComposeResult: + yield Static(self._top_text(), id="top") + yield VerticalScroll(id="chat") + # 提示放到边框标题里,不用文本区的 placeholder: + # 长占位符在部分终端增量重绘时不会被擦除,打字后会残留「后面还有字」, + # 直到全量重绘(Enter/截图/resize)才消失;边框标题在边框上,不受此影响。 + yield Input(id="prompt") + yield Static("", id="status") + + def on_mount(self) -> None: + inp = self.query_one("#prompt", Input) + inp.border_title = "输入消息 · Enter 发送 · Esc 中断 · /help" + # 加载完成前禁用输入框,避免用户在模型就绪前发消息 + inp.disabled = True + self._set_status("加载模型中…") + self._load() + + def on_input_changed(self, event: Input.Changed) -> None: + # 兜底:强制整屏重绘,清除个别终端增量重绘遗留的输入残影 + # (等价于截图/resize 触发的全量刷新)。 + self.refresh() + + def _top_text(self) -> Text: + t = Text() + t.append(LOGO, style=f"bold {_ACCENT}") + t.append(f" {_short_model(self.args.model)} · k={self.args.k} · " + f"{self.args.max_tokens} tok", style="dim") + return t + + def _set_status(self, text: str) -> None: + self.query_one("#status", Static).update( + f" {_short_model(self.args.model)} · {text}") + + @work(thread=True, exclusive=True, group="load") + def _load(self) -> None: + # 在 worker 线程执行阻塞加载,回调需经 call_from_thread 回到 UI 线程 + def on_status(msg: str): + # 应用已退出时不再回推,避免 call_from_thread 抛错 + if self.is_running: + self.call_from_thread(self._set_status, f"加载中 · {msg}") + + try: + self.backend.load(on_status) + except Exception as e: # noqa: BLE001 + if self.is_running: + self.call_from_thread(self._on_load_failed, str(e)) + return + if self.is_running: + self.call_from_thread(self._on_load_done) + + def _enable_input(self) -> None: + """重新启用并聚焦输入框(加载完成/一轮生成结束后统一调用)。 + + 刚把 disabled 置 False 时 can_focus 状态可能还没刷新,直接 focus 偶发无效 + (表现为输入框失焦时占位提示不消失);故延到下一次刷新后再聚焦,确保稳定拿到焦点。 + """ + inp = self.query_one("#prompt", Input) + inp.disabled = False + self.call_after_refresh(inp.focus) + + def _on_load_done(self) -> None: + self._set_status(f"就绪{_mem_suffix()}") + self._enable_input() + + def _on_load_failed(self, err: str) -> None: + self._add("assistant", f"模型加载失败:{err}\n\n请检查路径后用 /exit 退出重试。") + self._set_status("加载失败") + + def on_input_submitted(self, event: Input.Submitted) -> None: + text = event.value.strip() + event.input.value = "" + if not text: + return + if text.startswith("/"): + self._command(text) + return + if self._busy: + return + self._start(text) + + def _command(self, cmd: str) -> None: + if cmd in ("/exit", "/quit"): + self.exit() + elif cmd == "/help": + self._add("assistant", _HELP) + elif cmd in ("/reset", "/clear") and self._busy: + # 生成中改动历史/移除正在流式的 _cur 会让 _on_stream/_on_done 操作已卸载组件,故拒绝 + self._add("assistant", "生成中,请等本轮结束或按 Esc 中断后再执行该命令。") + elif cmd == "/reset": + # 只清空 system 之后的历史 + del self._messages[self._base_len:] + self._add("assistant", "对话历史已清空。") + elif cmd == "/clear": + self.query_one("#chat", VerticalScroll).remove_children() + else: + self._add("assistant", f"未知命令:{cmd}(/help 查看可用命令)") + + def _start(self, user_text: str) -> None: + self._messages.append({"role": "user", "content": user_text}) + self._add("user", user_text) + self._cur = self._add("assistant", "", final=False) + self._busy = True + self._stop = False + # 置 0 表示还没收到首 token;真正的计时起点推迟到第一次 _on_stream。 + self._gen_t0 = 0.0 + self._n0 = 0 + self._samples.clear() + self.query_one("#prompt", Input).disabled = True + self._set_status("思考中…") + self._generate(list(self._messages)) + + @work(thread=True, exclusive=True, group="gen") + def _generate(self, messages: list[dict]) -> None: + # 生成在 worker 线程运行;on_text 返回 self._stop 让后端可提前中断 + def on_text(full: str, n_tokens: int) -> bool: + # 应用已退出时,call_from_thread 会抛错;此时直接请求停止,避免 worker 线程未捕获异常。 + if not self.is_running: + return True + try: + self.call_from_thread(self._on_stream, full, n_tokens) + except Exception: # noqa: BLE001 + return True + return self._stop + + try: + result = self.backend.generate(messages, on_text) + except Exception as e: # noqa: BLE001 + if self.is_running: + self.call_from_thread(self._on_error, str(e)) + return + if self.is_running: + self.call_from_thread(self._on_done, result) + + def _record_sample(self, now: float, n_tokens: int) -> None: + """记录一次采样,并丢弃早于滑动窗口的旧点(保留跨越窗口边界的那一个)。""" + self._samples.append((now, n_tokens)) + w = self._tps_window + # 当第二个点仍比窗口还老时,第一个点已冗余,可丢弃; + # 循环后 _samples[0] 恰好是刚跨过窗口边界的采样,窗口长度 ≈ w。 + while len(self._samples) >= 2 and now - self._samples[1][0] >= w: + self._samples.popleft() + + def _window_tps(self, now: float) -> Optional[float]: + """按滑动窗口算瞬时 tok/s;样本不足(不够两点或时间差为 0)时返回 None。""" + if len(self._samples) < 2: + return None + t0, n0 = self._samples[0] + dt = now - t0 + if dt <= 0: + return None + return (self._samples[-1][1] - n0) / dt + + def _on_stream(self, full: str, n_tokens: int) -> None: + if self._cur is not None: + self._cur.stream(full) + self._scroll_end() + now = time.monotonic() + # 首次回调:prefill 刚结束,记下计时起点与基准 token 数,供结束时算累计解码平均。 + if self._gen_t0 == 0.0: + self._gen_t0 = now + self._n0 = n_tokens + # 生成中显示滑动窗口「瞬时」速度:一直在动,能反映后期变慢,不被历史平均拖住。 + self._record_sample(now, n_tokens) + tps = self._window_tps(now) + mem = _mem_suffix() + if tps is None: + self._set_status(f"思考中 · {n_tokens} tok{mem}") + else: + self._set_status(f"思考中 · {n_tokens} tok · {tps:.1f} tok/s{mem}") + + def _on_done(self, result: GenResult) -> None: + if self._cur is not None: + self._cur.finalize(result.text) + self._messages.append({"role": "assistant", "content": result.text}) + self._busy = False + self._cur = None + suffix = " · 已中断" if result.stopped else "" + # 与流式状态栏同口径:从首 token 起、按解码 token 数算,避免结束瞬间数字回落。 + # 若本轮没触发过流式(_gen_t0 仍为 0),退回后端上报的 tok/s。 + dt = time.monotonic() - self._gen_t0 + if self._gen_t0 > 0.0 and dt > 0: + tps = (result.n_tokens - self._n0) / dt + else: + tps = result.tok_per_s + self._set_status( + f"就绪 · {result.n_tokens} tok · {tps:.1f} tok/s{_mem_suffix(peak=True)}{suffix}") + self._enable_input() + + def _on_error(self, err: str) -> None: + if self._cur is not None: + self._cur.finalize(f"生成出错:{err}") + self._busy = False + self._cur = None + self._set_status("就绪(上一轮出错)") + self._enable_input() + + def action_interrupt(self) -> None: + # 仅在生成中时置位中断标志,由 on_text 闭包读取 + if self._busy: + self._stop = True + self._set_status("正在中断…") + + def _add(self, role: str, text: str, *, final: bool = True) -> ChatMessage: + msg = ChatMessage(role, text, final=final) + self.query_one("#chat", VerticalScroll).mount(msg) + self._scroll_end() + return msg + + def _scroll_end(self) -> None: + self.query_one("#chat", VerticalScroll).scroll_end(animate=False) diff --git a/mlx_streaming/tui/backend.py b/mlx_streaming/tui/backend.py new file mode 100644 index 0000000..c7a6471 --- /dev/null +++ b/mlx_streaming/tui/backend.py @@ -0,0 +1,436 @@ +"""TUI 后端抽象:把界面与 MLX 推理引擎解耦。 + +界面只依赖 ChatBackend 接口,不 import 任何 MLX 符号,从而能用 FakeBackend 做无模型测试。 +load / generate 都是阻塞调用,由 UI 层放到 worker 线程执行。 +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Callable, Protocol +from collections import OrderedDict + + +@dataclass +class GenResult: + """一轮生成的汇总。""" + + text: str # 完整回答(已截断 EOS) + n_tokens: int # 新生成 token 数 + tok_per_s: float # 吞吐 + stopped: bool # 是否被用户中断 + + +class ChatBackend(Protocol): + """聊天后端接口。所有方法阻塞,调用方负责放 worker 线程。""" + + def load(self, on_status: Callable[[str], None]) -> None: + """加载模型/权重;通过 on_status(msg) 上报进度。""" + ... + + def generate( + self, + messages: list[dict], + on_text: Callable[[str, int], bool], + ) -> GenResult: + """跑一轮生成。每步把「累计完整文本, 已生成 token 数」传给 on_text;返回 True 表示请求中断。""" + ... + + +@dataclass +class FakeBackend: + """测试用假后端:不加载模型,把预设回答按字符流式吐出。 + + delay > 0 时每字符间 sleep,用于 --demo 模式模拟真实吐字节奏;测试默认 0(不拖慢)。 + """ + + reply: str = "你好，这是一个测试回答。" + status_msgs: list[str] = field(default_factory=lambda: ["加载中(模拟)…"]) + delay: float = 0.0 + seen_messages: list[list[dict]] = field(default_factory=list) + + def load(self, on_status: Callable[[str], None]) -> None: + for m in self.status_msgs: + on_status(m) + + def generate(self, messages, on_text, tools=None, sampling=None) -> GenResult: + import time + + self.seen_messages.append([dict(m) for m in messages]) + acc = "" + for ch in self.reply: + acc += ch + if self.delay: + time.sleep(self.delay) + # 用字符数近似 token 数(假后端无真实分词) + if on_text(acc, len(acc)): + return GenResult(acc, len(acc), 0.0, stopped=True) + return GenResult(acc, len(self.reply), 0.0, stopped=False) + + +def _common_prefix_len(a, b) -> int: + """返回两个 token id 序列的最长公共前缀长度。""" + n = min(len(a), len(b)) + i = 0 + while i < n and a[i] == b[i]: + i += 1 + return i + + +def _reuse_prefix_len(cached_ids, new_ids) -> int: + """可跨轮复用的前缀长度:仅当旧 cache 的 token 是新序列的**严格前缀**且新序列更长时, + 返回该前缀长度(= len(cached_ids));否则返回 0 表示需全量重建。 + + 只在严格前缀时复用,是为了永远「只延续、不回退」cache——Qwen3-Next 的线性注意力递归态 + 无法裁剪回任意历史位置;而 detokenize→retokenize 不一致、/reset、编辑历史等都会让公共前缀 + 短于旧长度,此时回退整段重建,绝不基于错位的 cache 续算。 + """ + if not cached_ids or len(new_ids) <= len(cached_ids): + return 0 + c = _common_prefix_len(cached_ids, new_ids) + return c if c == len(cached_ids) else 0 + + +class MLXBackend: + """真实后端:封装 _build_engine + mtp_generate。MLX 相关 import 全部延迟到方法内。""" + + def __init__(self, args): + self.args = args + self._model = None + self._tok = None + self._drafter = None + # 跨轮复用:持久化上一轮的 main_cache 及其对应的 token 序列(prompt + 已入 cache 的生成 token)。 + self._main_cache = None + self._cached_ids: list[int] = [] + # 跨会话前缀快照:{head_key: (snaps, head_ids)},LRU。agentic 场景每个新会话 + # 都重复同一段长 system+tools 前缀,快照后首轮 prefill 从 ~2600 tok 降到几百。 + self._head_snaps: "OrderedDict[tuple, tuple]" = OrderedDict() + # (system 文本, 工具名) → 固定头部 token 数,见 _fixed_prefix_len + self._fixed_head_key = None + self._fixed_head_len = 0 + # 流式反分词器(单例,每轮 reset)。建实例要铺一张 15 万项的 id→token 表,不能每轮重建; + # 生成由 server 的单飞锁串行,单例安全。 + self._detok = None + + def load(self, on_status: Callable[[str], None]) -> None: + from mlx_streaming.cli import _build_engine, _warmup + + self._model, self._tok, self._drafter = _build_engine( + self.args, on_status=on_status + ) + # 预热:把首轮的 kernel 编译 + 专家池填充开销移到加载阶段,避免第一条消息莫名卡很久。 + on_status("预热中(编译 kernel + 填专家池)…") + _warmup(self._model, self._tok, self._drafter, self.args) + # 采样参数:读模型自带 generation_config(Qwen3-Next 官方要求 do_sample + # temp=0.7/top_p=0.8/top_k=20;贪心 argmax 会诱发幻觉/编造,见 2026-07 排查) + import json + import os + self._gen_cfg = {"temperature": 0.7, "top_p": 0.8, "top_k": 20} + try: + with open(os.path.join(str(self.args.model), "generation_config.json")) as f: + gc = json.load(f) + if gc.get("do_sample", True): + for k in ("temperature", "top_p", "top_k"): + if k in gc: + self._gen_cfg[k] = gc[k] + except OSError: + pass + + def _generate_sampled(self, ids, on_tokens, max_tokens, sampling, + main_cache, cached_len): + """采样生成(非投机):prefill 后逐 token 前向 + temp/top_p/top_k 采样。 + 返回 (produced, forwarded):forwarded = 已写入 cache 的生成 token 数, + 供跨轮复用不变式(resident == len(ids)+forwarded)判定。 + + Qwen3-Next 的 generation_config 要求 do_sample;贪心 argmax 在 agentic 长 + 提示下会编造工具结果/幻觉工具名(实测)。代价:无 MTP 加速,decode 约 4-6 tok/s。 + """ + import mlx.core as mx + from mlx_lm.sample_utils import make_sampler + from mlx_streaming.mtp.generate import forward_with_hidden, prefill_chunked + + sampler = make_sampler( + temp=float(sampling.get("temperature", 0.7)), + top_p=float(sampling.get("top_p") or 0.0), + top_k=int(sampling.get("top_k") or 0)) + + def _logprobs(logits): + """末位 logits → float32 归一化 log-probs,这是 mlx_lm sampler 的入参契约。 + + 必须归一化:apply_top_p 里是 `probs = mx.exp(x)` 再与 `1 - top_p` 比累积和, + 喂未归一化的 logits 会让 exp() 变成天文数字、累积和瞬间越过阈值 → top_p 完全失效 + (只砍掉概率 <1e-12 的尾巴)。必须转 float32:lm_head 输出是 bf16(8 位尾数), + exp/cumsum/categorical 在 bf16 里做会把分布压得面目全非。 + """ + lg = logits[:, -1, :].astype(mx.float32) + return lg - mx.logsumexp(lg, axis=-1, keepdims=True) + + ids_mx = ids if isinstance(ids, mx.array) else mx.array([ids]) + logits, _ = prefill_chunked(self._model, ids_mx[:, cached_len:], main_cache) + cur = _logprobs(logits) + produced = [] + forwarded = 0 + for _ in range(max_tokens): + nxt = int(sampler(cur)) + produced.append(nxt) + if on_tokens(produced[-1:]): + break # 停止:当前 token 未写入 cache + logits, _ = forward_with_hidden(self._model, mx.array([[nxt]]), main_cache) + mx.eval(logits) + forwarded += 1 + cur = _logprobs(logits) + return produced, forwarded + + # ---- 前缀快照磁盘持久化 ---- + # 目录:models/prefix_snapshots/{key}.safetensors + {key}.json(sidecar:head ids/模型名)。 + # 首个会话 prefill 一次落盘;之后(含引擎重启后)同前缀会话直接读盘恢复,消灭"首次"。 + + def _snap_dir(self) -> str: + import os + d = os.path.join(os.path.dirname(str(self.args.expert_dir)) or ".", "prefix_snapshots") + os.makedirs(d, exist_ok=True) + return d + + @staticmethod + def _snap_key(ids, head: int) -> str: + import hashlib + import numpy as np + return hashlib.sha1(np.array(ids[:head], dtype=np.int32).tobytes()).hexdigest()[:16] + + def _snap_save(self, key: str, snaps, ids, head: int) -> None: + import json + import os + + import mlx.core as mx + from mlx.utils import tree_flatten + + from mlx_streaming import config as _cfg + + flat = {} + metas = [] + for ci, (st, meta) in enumerate(snaps): + metas.append(list(meta) if isinstance(meta, (tuple, list)) else (meta or "")) + for k, v in tree_flatten(st): + flat[f"c{ci}.{k}"] = v + path = os.path.join(self._snap_dir(), key) + mx.save_safetensors(path + ".safetensors", flat) + with open(path + ".json", "w") as f: + # kv_quant 决定 cache 类型(量化/非量化),快照跨配置复用会 500 或错算,必须校验 + json.dump({"head": head, "model": os.path.basename(str(self.args.model)), + "kv_quant": _cfg.kv_quant(), + "metas": metas, "ids": list(ids[:head])}, f) + + def _snap_load(self, key: str, ids, head: int): + """磁盘命中返回 snaps(供 _cache_restore);缺失/模型不符/损坏返回 None。""" + import json + import os + + import mlx.core as mx + from mlx.utils import tree_flatten, tree_unflatten + + path = os.path.join(self._snap_dir(), key) + try: + with open(path + ".json") as f: + side = json.load(f) + if side.get("head") != head or side.get("ids") != list(ids[:head]): + return None + if side.get("model") != os.path.basename(str(self.args.model)): + return None + # kv_quant 配置必须一致:量化 KV 的快照装进非量化 cache(或反之)会崩/错算 + from mlx_streaming import config as _cfg + if bool(side.get("kv_quant", True)) != bool(_cfg.kv_quant()): + return None + flat = mx.load(path + ".safetensors") + metas = side.get("metas") or [] + n_cache = max(int(k.split(".", 1)[0][1:]) for k in flat) + 1 + snaps = [] + for ci in range(n_cache): + sub = {k.split(".", 1)[1]: v for k, v in flat.items() + if k.startswith(f"c{ci}.")} + meta = metas[ci] if ci < len(metas) else "" + if isinstance(meta, list): + meta = tuple(meta) + snaps.append((tree_unflatten(list(sub.items())), meta)) + mx.eval([v for st, _ in snaps for v in tree_flatten(st)]) + return snaps + except (OSError, ValueError, KeyError, json.JSONDecodeError): + return None + + def _fixed_prefix_len(self, messages, tools) -> int: + """算出 prompt 里「与用户问题无关」的固定头部长度(system + tools 渲染后的 token 数)。 + + 做法是把同样的 system/tools 配一个哑 user 再渲染一次,取两次渲染的公共前缀—— + 分叉点就是 user 内容的起始位置。这样得到的边界与提问内容无关,是快照能命中的最大头部: + 再多一个 token 就把问题本身包进快照,换个问法即失效;少了则白白重算已知不变的部分 + (实测 system+4 工具 = 3972 token,固定切 2048 会让每个新会话多 prefill 1900+ token)。 + + 结果按 (system 文本, 工具名集合) 缓存:每轮只在配置变化时付一次 tokenize。 + """ + from mlx_streaming.cli import _encode_chat + + head_msgs = [m for m in messages if m.get("role") == "system"] + if not head_msgs: + return 0 + key = ("".join(m.get("content") or "" for m in head_msgs), + tuple(sorted((t.get("function") or {}).get("name", "") + for t in (tools or [])))) + if getattr(self, "_fixed_head_key", None) == key: + return self._fixed_head_len + probe = _encode_chat(self._tok, head_msgs + [{"role": "user", "content": "\x00probe"}], + tools=tools) + real = _encode_chat(self._tok, head_msgs + [{"role": "user", "content": "\x01other"}], + tools=tools) + n = 0 + for a, b in zip(probe, real): + if a != b: + break + n += 1 + self._fixed_head_key, self._fixed_head_len = key, n + return n + + def _snapshot_prefix(self, ids, fixed_head: int = 0): + """跨会话前缀快照。内存命中/磁盘命中:恢复到新 cache 返回 (cache, head); + 未命中:prefill 头部、深拷贝存内存 + 落盘后返回 (cache, head) + (首个会话不增加前向量,只是把整段 prefill 拆两段);短 prompt 或关闭时返回 (None, 0)。 + + head 取 min(PREFIX_SNAPSHOT_HEAD 上限, system+tools 实际长度):快照必须停在 + 用户问题之前才能被后续会话命中,而停得越晚省下的 prefill 越多。 + """ + import mlx.core as mx + + from mlx_streaming import config + from mlx_streaming.mtp.generate import prefill_chunked + from mlx_streaming.mtp.kv_cache import _restore as _cache_restore + from mlx_streaming.mtp.kv_cache import _snapshot as _cache_snapshot + + head = config.prefix_snapshot_head() + if fixed_head > 0: + head = min(head, fixed_head) + head = min(head, len(ids) - 1) # 至少留 1 个 token 走前向,否则拿不到 logits + # 门槛按 head 自身算,不按「prompt 比 head 长多少」:快照省下的就是 head 这段, + # 与后面剩多少无关。头部 3972、问题只有 15 token 恰恰是收益最大的情形。 + if head < 256: + return None, 0 + key = self._snap_key(ids, head) + ent = self._head_snaps.get(key) + if ent is not None: + cache = self._model.make_cache() + _cache_restore(cache, ent) + self._head_snaps.move_to_end(key) + return cache, head + snaps = self._snap_load(key, ids, head) + if snaps is not None: + cache = self._model.make_cache() + _cache_restore(cache, snaps) + self._head_snaps[key] = snaps + self._head_snaps.move_to_end(key) + return cache, head + cache = self._model.make_cache() + prefill_chunked(self._model, mx.array([ids[:head]]), cache) + snaps = _cache_snapshot(cache) + self._head_snaps[key] = snaps + self._head_snaps.move_to_end(key) + while len(self._head_snaps) > config.prefix_snapshot_entries(): + self._head_snaps.popitem(last=False) + try: + self._snap_save(key, snaps, ids, head) + except Exception: # noqa: BLE001 落盘失败不影响主路径(内存快照仍生效) + pass + return cache, head + + def generate(self, messages, on_text, tools=None, sampling=None) -> GenResult: + import time + + import mlx.core as mx + + from mlx_streaming.cli import _encode_chat, _eos_set, _truncate_eos + from mlx_streaming.mtp.generate import mtp_generate + + tok = self._tok + eos = _eos_set(tok) + ids = _encode_chat(tok, messages, tools=tools) + + # 跨轮复用 KV/递归态:旧 cache 是本轮 prompt 的严格前缀时,只 prefill 新增后缀, + # 不重算整段历史(prefill 从 ∝历史长度 降到 ∝新消息长度)。否则全量重建。 + cached_len = (_reuse_prefix_len(self._cached_ids, ids) + if self._main_cache is not None else 0) + main_cache = self._main_cache if cached_len else None + + # 跨会话前缀快照:同一段长 system+tools 头部的新会话,跳过头部 prefill。 + # 首个带该头部的会话在 prefill 头部后深拷贝一份状态存下(不产生额外前向, + # 只是把一个整段 prefill 拆成两段),后续会话直接恢复。 + if main_cache is None: + main_cache, cached_len = self._snapshot_prefix( + ids, fixed_head=self._fixed_prefix_len(messages, tools)) + + if main_cache is None: + main_cache = self._model.make_cache() + cached_len = 0 + + # 流式反分词:必须逐 token 喂 StreamingDetokenizer,不能每步 tok.decode(全部已产 token)。 + # Qwen 是 byte-level BPE,很多汉字/符号跨 2 个 token,整体 decode 到半个字符时会按 + # errors="replace" 吐出 U+FFFD;而下游按字符数算增量(delta = full[len(prev):]),下一步 + # U+FFFD 被真字符替换后长度不变 → 增量为空、真字符永久丢失,客户端就留下一堆「�」。 + # detokenizer 的 .text 只包含已完整的字符(未完成字节留在 _unflushed),恒为下一步的真前缀。 + detok = self._detok + if detok is None: + detok = self._detok = tok.detokenizer + detok.reset() + produced_all: list[int] = [] # 已上报的正文 token(不含 EOS 及其后) + stopped = {"v": False} + hit_eos = {"v": False} + + def on_tokens(new_ids): + for t in new_ids: + if int(t) in eos: + hit_eos["v"] = True + break + detok.add_token(int(t)) + produced_all.append(int(t)) + if hit_eos["v"]: + detok.finalize() # 冲出尾部残留字节(正常收尾,不会有半字) + if on_text(detok.text, len(produced_all)): # 用户按 Esc 请求中断 + stopped["v"] = True + return True + # 命中 EOS:完整回答已生成,提前停止,避免引擎空跑到 max_tokens 让界面 + # 长时间卡在「思考中」。EOS 属正常完成,不算中断。 + return hit_eos["v"] + + t0 = time.perf_counter() + if sampling is not None: + # 采样模式(server 默认,质量优先):非投机逐 token 采样,无 MTP 加速。 + produced, forwarded = self._generate_sampled( + ids, on_tokens, self.args.max_tokens, sampling, + main_cache, cached_len) + stats = {"resident_tokens": len(ids) + forwarded} + else: + produced, stats = mtp_generate( + self._model, + self._drafter, + tok, + mx.array([ids]), + self.args.max_tokens, + K=self.args.k, + ids_mode=True, + profile=False, + on_tokens=on_tokens, + main_cache=main_cache, + cached_len=cached_len, + ) + dt = time.perf_counter() - t0 + + # 持久化本轮 cache 供下轮复用。正常情况下 main_cache 恰好持有 `ids + produced[:-1]` + # (produced[-1] 为 pending 未入 cache)。但末步多 token 跨 max_tokens 会 over-commit: + # cache 领先于 produced,无法用已知 token 精确表述——此时禁用复用,下轮全量重建,绝不错算。 + resident = stats.get("resident_tokens") + expected = len(ids) + len(produced) - 1 + if resident == expected: + self._main_cache = main_cache + self._cached_ids = list(ids) + list(produced[:-1]) + else: + self._main_cache = None + self._cached_ids = [] + + out_ids = _truncate_eos(produced, eos) + text = tok.decode(out_ids) + tps = len(out_ids) / dt if dt > 0 else 0.0 + return GenResult(text, len(out_ids), tps, stopped=stopped["v"]) diff --git a/mlx_streaming/tui/banner.py b/mlx_streaming/tui/banner.py new file mode 100644 index 0000000..0f85b06 --- /dev/null +++ b/mlx_streaming/tui/banner.py @@ -0,0 +1,3 @@ +"""sparkle 顶栏 logo。保持单行,便于与模型信息拼在同一行。""" + +LOGO = "▚ sparkle" diff --git a/mlx_streaming/tui/styles.tcss b/mlx_streaming/tui/styles.tcss new file mode 100644 index 0000000..de1168a --- /dev/null +++ b/mlx_streaming/tui/styles.tcss @@ -0,0 +1,41 @@ +Screen { + background: #0b0e14; +} + +#top { + height: auto; + padding: 1 2; + background: #11151f; + color: #e6e6e6; + border-bottom: solid #2a2f3a; +} + +#chat { + padding: 1 2; + height: 1fr; +} + +.msg { + margin: 1 0; + padding: 0 1; + height: auto; +} + +.msg.assistant { + border-left: solid #2dd4bf; + padding-left: 2; +} + +#prompt { + margin: 0 2; + border: round #2dd4bf; + border-title-color: #8b93a7; + background: #11151f; +} + +#status { + height: 1; + padding: 0 2; + background: #11151f; + color: #8b93a7; +} diff --git a/models/README.md b/models/README.md new file mode 100644 index 0000000..56063a8 --- /dev/null +++ b/models/README.md @@ -0,0 +1,49 @@ +# 模型与专家数据（不入库，请本机下载/生成） + +## 为什么要做 + +目标不是「把 80B 常驻塞进 64G」，而是用本机 **64G 统一内存** 做压力验证：把约 **77G 专家权重** 以流式专家池压到约 **33G 常驻**，在 8bit 下跑通 Qwen3-Next-80B-A3B。 + +验证点对应下游的 **黄金拐点**：Qwen3 **30B 8bit A3B** 一类模型，用同一套流式装载思路，能放进 **32G / 16G** 统一内存机器，相对「整模进内存」可 **节省 50% 以上统一内存**。本仓库的 80B @ 64G 是这条路径的上限探针；量产形态落在更小机型上的 30B 拐点模型。 + +--- + +## 运行依赖版本 + +权重本身不绑版本，但本仓库预编译扩展与锁文件对齐如下（换版本可能无法 `import native_moe_ext` 或加载 8bit）： + +| 项 | 要求 / 已验证 | +|----|----------------| +| Python | **3.14.***（已验证 3.14.6；须匹配 `native_moe_ext.cpython-314-darwin.so`） | +| mlx | `>=0.31`（锁文件 **0.31.2**） | +| mlx-lm | `>=0.31`（锁文件 **0.31.3**） | +| numpy | `>=2.0`（锁文件 **2.4.6**） | +| fastapi | `>=0.115`（锁文件 **0.140.0**） | +| uvicorn | `>=0.30`（锁文件 **0.51.0**） | +| textual | `>=0.80`（锁文件 **8.2.8**） | + +安装：在仓库根目录 `uv sync`（或使用已有 `.venv`）。完整约束见根目录 `pyproject.toml` / `uv.lock`。换 Python 小版本需 `make -C native/ext native_moe_ext` 重编 `.so`。 + +权重体积约 **160GB**，不进 Git。按下面准备后目录应类似： + +``` +models/ +├── Qwen3-Next-80B-A3B-Instruct-MLX-8bit/ # ~79GB 主模型 +├── experts_8bit_g64/ # ~77GB 专家 blob（运行时常驻约 33G） +└── qn_mtp_weights.safetensors # ~3.1GB MTP +``` + +## 下载（NAS） + +全部模型与专家数据从 NAS 共享下载： + +- 链接：https://ug.link/dxp4800plus-8761/filemgr/share-download/?id=86b004b90f4a477e84c7f65c87943b2f +- 密码：`sparkle` + +解压/放置到本仓库 `models/` 下，目录结构与上文一致即可。 + +> 说明：本仓库当前跑通的是 **80B** 探针；包内若含 **Qwen3-30B-A3B-Instruct-2507-MLX-8bit**，供下游 16G/32G 机型对照，不替代本机 80B 数据准备。 + +## 可选 + +`prefix_snapshots/` 会在首次跑通后自动生成，可不预置。 diff --git a/native/ext/CMakeLists.txt b/native/ext/CMakeLists.txt new file mode 100644 index 0000000..2df3844 --- /dev/null +++ b/native/ext/CMakeLists.txt @@ -0,0 +1,32 @@ +cmake_minimum_required(VERSION 3.22) +project(native_moe_mlx_ext LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +find_package(Python COMPONENTS Interpreter Development.Module REQUIRED) +find_package(nanobind CONFIG REQUIRED) +find_package(MLX CONFIG REQUIRED) + +nanobind_add_module( + native_moe_ext + NB_STATIC + STABLE_ABI + LTO + NOMINSIZE + NB_DOMAIN mlx + compute/fused_moe.cpp + io/blob_load.cpp + io/bg_reader.cpp + prefetch/prefetch.cpp + pool/owned_pool.cpp + pool/side_region.cpp + pool/demand.cpp + bindings.cpp +) + +target_link_libraries(native_moe_ext PRIVATE mlx) + +if(BUILD_SHARED_LIBS) + target_link_options(native_moe_ext PRIVATE -Wl,-rpath,@loader_path) +endif() diff --git a/native/ext/Makefile b/native/ext/Makefile new file mode 100644 index 0000000..9ecea0e --- /dev/null +++ b/native/ext/Makefile @@ -0,0 +1,14 @@ +# 生产 MLX 扩展构建：cmake 把 native_moe_mlx_ext.cpp 编成 +# mlx_streaming/native_moe_ext$(EXT_SUFFIX).so（nanobind + MLX Primitive）。 +PYTHON ?= ../../.venv/bin/python +PY_SITE := $(abspath ../../.venv/lib/python3.12/site-packages) +NANOBIND_DIR := $(shell $(PYTHON) -m nanobind --cmake_dir) +MLX_CMAKE_DIR := $(PY_SITE)/mlx/share/cmake/MLX + +native_moe_ext: + cmake -S . -B build -DPython_EXECUTABLE=$(abspath $(PYTHON)) -Dnanobind_DIR=$(NANOBIND_DIR) -DMLX_DIR=$(MLX_CMAKE_DIR) -DCMAKE_LIBRARY_OUTPUT_DIRECTORY=$(abspath ../../mlx_streaming) + cmake --build build --target native_moe_ext -j + +clean: + rm -rf build + rm -f ../../mlx_streaming/native_moe_ext*.so diff --git a/native/ext/bindings.cpp b/native/ext/bindings.cpp new file mode 100644 index 0000000..99237a9 --- /dev/null +++ b/native/ext/bindings.cpp @@ -0,0 +1,153 @@ +#include "compute/fused_moe.h" +#include "io/blob_load.h" +#include "io/bg_reader.h" +#include "prefetch/prefetch.h" +#include "pool/owned_pool.h" +#include "pool/side_region.h" +#include "pool/demand.h" +using namespace nb::literals; + +NB_MODULE(native_moe_ext, m) { + nb::module_::import_("mlx.core"); + m.doc() = "Native MLX MoE extension."; + + // ---- 融合 MoE 计算核（native_fused.cpp）---- + m.def( + "fused_moe", + &fused_moe, + "x"_a, + "expert_ids"_a, + "scores"_a, + "compute_dir"_a, + "layer"_a, + "hidden"_a, + "inter"_a, + "group"_a, + "bits"_a, + "num_experts"_a, + "synthetic"_a, + nb::kw_only(), + "stream"_a = nb::none()); + m.def( + "fused_moe_staged", + &fused_moe_staged, + "x"_a, + "scores"_a, + "gate_w"_a, + "gate_s"_a, + "gate_b"_a, + "up_w"_a, + "up_s"_a, + "up_b"_a, + "down_w"_a, + "down_s"_a, + "down_b"_a, + "hidden"_a, + "inter"_a, + "group"_a, + "bits"_a, + nb::kw_only(), + "stream"_a = nb::none()); + // ---- [1] blob 直读 ---- + m.def( + "blob_load", + &blob_load, + "path"_a, + "expert_ids"_a, + "stride"_a, + nb::kw_only(), + "stream"_a = nb::none()); + // ---- [2] 轻量预取(无 staging，仅预热 page cache) ---- + m.def( + "prefetch_on_complete", + &prefetch_on_complete, + "expert_ids"_a, + "path"_a, + "stride"_a, + "do_read"_a = true, + nb::kw_only(), + "stream"_a = nb::none()); + // ---- [3] staging miss→hit + 完成回调时刻探针（诊断）---- + m.def( + "prefetch_into_staging", + &prefetch_into_staging, + "staging"_a, + "expert_ids"_a, + "layer"_a, + "gen"_a, + "path"_a, + "stride"_a, + "resident"_a, + "cap"_a, + "parallel"_a, + nb::kw_only(), + "stream"_a = nb::none()); + m.def("prefetch_staging_take", &prefetch_staging_take, "layer"_a); + m.def("staging_hprof_enable", &staging_hprof_enable, "on"_a); + m.def("staging_hprof_now", &staging_hprof_now); + m.def("staging_hprof_get", &staging_hprof_get); + // ---- [4] 段散写侧区缓存（zero-copy dual-source 默认路径）---- + m.def("prefetch_pool_sideregion", &prefetch_pool_sideregion, + "pool_list"_a, "seg_nbytes"_a, "expert_ids"_a, "layer"_a, "path"_a, "stride"_a, + "resident"_a, "spec_slots"_a, "base_row"_a, "gen"_a = 0, nb::kw_only(), + "stream"_a = nb::none()); + m.def("sideregion_contents", &sideregion_contents, "layer"_a, "gen"_a = 0); + m.def("sideregion_kv", &sideregion_kv, "layer"_a, "gen"_a = 0); + m.def("sideregion_reset", &sideregion_reset); + m.def("sideregion_drain", &sideregion_drain, + nb::call_guard()); // 等待时释放 GIL + // ---- [5] Route 3 owned 池底座（C++ 拥有 buffer + 直写，删 MLX scatter）---- + m.def("pool_owned_zeros", &pool_owned_zeros, "shape"_a, "dtype"_a); + m.def("pool_write_rows", &pool_write_rows, "pool_list"_a, "srcs_flat"_a, "slots"_a); + m.def("pool_write_stacked", &pool_write_stacked, "pool_list"_a, "stacked_list"_a, "slots"_a); + m.def("array_data_ptr", &array_data_ptr, "a"_a); + // ---- [6] 方案B 真实区槽状态 C++ 接管 + demand_dual ---- + m.def("real_init", &real_init, "layer"_a, "cap"_a); + m.def("real_region_contents", &real_region_contents, "layer"_a); + m.def("real_region_count", &real_region_count, "layer"_a); + m.def("real_reset", &real_reset); + m.def("real_pin", &real_pin, "layer"_a, "experts"_a); + m.def("real_pinned_count", &real_pinned_count, "layer"_a); + m.def("real_freq_dump", &real_freq_dump); + m.def("demand_dual", &demand_dual, "inds"_a, "pool_list"_a, "seg_nbytes"_a, "layer"_a, + "side_gen"_a, "path"_a, "stride"_a, "cap"_a, "lfu"_a, "decay_interval"_a, nb::kw_only(), + "stream"_a = nb::none()); + m.def("demand_last_stats", &demand_last_stats); + m.def("demand_timings", &demand_timings); + m.def("demand_timing_enable", &demand_timing_enable, "on"_a); + m.def("real_debug_place", &real_debug_place, "layer"_a, "experts_flat"_a, "cap"_a, "lfu"_a, + "decay_interval"_a); + // ---- [7] 自由后台读线程 ---- + m.def("bg_reader_start", &bg_reader_start, "workers"_a = 1, "low_cap"_a = 0); + m.def("bg_reader_submit", &bg_reader_submit, + "dst"_a, "experts"_a, "rows"_a, "path"_a, "stride"_a, "ticket"_a, "prio"_a = 0); + m.def("bg_reader_ready", &bg_reader_ready, "ticket"_a); + m.def("bg_reader_wait", &bg_reader_wait, "ticket"_a, + nb::call_guard()); // 阻塞等时释放 GIL + m.def("bg_reader_stop", &bg_reader_stop); + m.def("bg_pread_into_pool", &bg_pread_into_pool, + "dst"_a, "seg_off"_a, "seg_nb"_a, "slot"_a, "expert"_a, + "path"_a, "stride"_a, "ticket"_a, "prio"_a = 0, "nocache"_a = true); + // ---- 融合 MoE 计算核（slots 版，native_fused.cpp）---- + m.def( + "fused_moe_slots", + &fused_moe_slots, + "x"_a, + "local_slots"_a, + "scores"_a, + "gate_w"_a, + "gate_s"_a, + "gate_b"_a, + "up_w"_a, + "up_s"_a, + "up_b"_a, + "down_w"_a, + "down_s"_a, + "down_b"_a, + "hidden"_a, + "inter"_a, + "group"_a, + "bits"_a, + nb::kw_only(), + "stream"_a = nb::none()); +} diff --git a/native/ext/common.h b/native/ext/common.h new file mode 100644 index 0000000..495cd0c --- /dev/null +++ b/native/ext/common.h @@ -0,0 +1,34 @@ +// 共用前导：系统/Metal/MLX/nanobind 头 + 命名空间别名。所有 TU 都包含。 +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mlx/backend/metal/device.h" +#include "mlx/mlx.h" +#include "mlx/primitives.h" + +namespace nb = nanobind; +namespace mx = mlx::core; diff --git a/native/ext/compute/fused_moe.cpp b/native/ext/compute/fused_moe.cpp new file mode 100644 index 0000000..0a94b0d --- /dev/null +++ b/native/ext/compute/fused_moe.cpp @@ -0,0 +1,664 @@ +#include "fused_moe.h" + +struct FusedParams { + int experts; + int hidden; + int inter; + int group_size; + int bits; + int k; +}; + +struct MappedProj { + void* w{MAP_FAILED}; + void* s{MAP_FAILED}; + void* b{MAP_FAILED}; + size_t wn{0}, sn{0}, bn{0}; + int wfd{-1}, sfd{-1}, bfd{-1}; +}; + +struct ProjectionShape { + int out_dim; + int in_dim; + size_t weight_per; + size_t scale_per; +}; + +static size_t file_size(int fd) { + struct stat st {}; + if (fstat(fd, &st) != 0) { + throw std::runtime_error("fstat failed"); + } + return static_cast(st.st_size); +} + +static void* map_file(const std::string& path, size_t& nbytes, int& fd) { + fd = open(path.c_str(), O_RDONLY); + if (fd < 0) { + throw std::runtime_error("failed to open " + path); + } + nbytes = file_size(fd); + void* p = mmap(nullptr, nbytes, PROT_READ, MAP_PRIVATE, fd, 0); + if (p == MAP_FAILED) { + throw std::runtime_error("failed to mmap " + path); + } + return p; +} + +static MappedProj map_proj(const std::string& dir, int layer, const std::string& proj) { + MappedProj m; + std::string layer_name = "layer" + std::string(layer < 10 ? "0" : "") + std::to_string(layer); + std::string base = dir + "/" + layer_name + "." + proj; + m.w = map_file(base + ".weight.bin", m.wn, m.wfd); + m.s = map_file(base + ".scales.bin", m.sn, m.sfd); + m.b = map_file(base + ".biases.bin", m.bn, m.bfd); + return m; +} + +static void unmap_proj(MappedProj& m) { + if (m.w != MAP_FAILED) munmap(m.w, m.wn); + if (m.s != MAP_FAILED) munmap(m.s, m.sn); + if (m.b != MAP_FAILED) munmap(m.b, m.bn); + if (m.wfd >= 0) close(m.wfd); + if (m.sfd >= 0) close(m.sfd); + if (m.bfd >= 0) close(m.bfd); +} + +static ProjectionShape proj_shape(int out_dim, int in_dim, int group, int bits) { + int words = (in_dim * bits) / 32; + int groups = in_dim / group; + return ProjectionShape{ + out_dim, + in_dim, + static_cast(out_dim) * words * sizeof(uint32_t), + static_cast(out_dim) * groups * sizeof(uint16_t), + }; +} + +static uint16_t fp32_to_bf16(float x) { + uint32_t u = 0; + std::memcpy(&u, &x, sizeof(u)); + return static_cast(u >> 16); +} + +static void fill_synthetic( + MTL::Buffer* wbuf, + MTL::Buffer* sbuf, + MTL::Buffer* bbuf, + uint32_t seed) { + std::mt19937 rng(seed); + auto* w = reinterpret_cast(wbuf->contents()); + size_t wn = wbuf->length() / sizeof(uint32_t); + for (size_t i = 0; i < wn; ++i) w[i] = rng(); + auto* s = reinterpret_cast(sbuf->contents()); + auto* b = reinterpret_cast(bbuf->contents()); + size_t sn = sbuf->length() / sizeof(uint16_t); + uint16_t scale = fp32_to_bf16(0.001f); + uint16_t bias = fp32_to_bf16(-0.032f); + for (size_t i = 0; i < sn; ++i) { + s[i] = scale; + b[i] = bias; + } +} + +static void copy_projection( + const MappedProj& src, + const ProjectionShape& shape, + const std::vector& ids, + MTL::Buffer* wbuf, + MTL::Buffer* sbuf, + MTL::Buffer* bbuf) { + auto* wd = static_cast(wbuf->contents()); + auto* sd = static_cast(sbuf->contents()); + auto* bd = static_cast(bbuf->contents()); + auto* ws = static_cast(src.w); + auto* ss = static_cast(src.s); + auto* bs = static_cast(src.b); + for (size_t local = 0; local < ids.size(); ++local) { + size_t expert = static_cast(ids[local]); + std::memcpy(wd + local * shape.weight_per, ws + expert * shape.weight_per, shape.weight_per); + std::memcpy(sd + local * shape.scale_per, ss + expert * shape.scale_per, shape.scale_per); + std::memcpy(bd + local * shape.scale_per, bs + expert * shape.scale_per, shape.scale_per); + } +} + +static std::string metal_source() { + return R"( + #include + using namespace metal; + struct FusedParams { int experts; int hidden; int inter; int group_size; int bits; int k; }; + inline float bf16_to_float(ushort v) { uint u = uint(v) << 16; return as_type(u); } + inline float qvalue(const device uint* w, const device ushort* s, const device ushort* b, + uint expert, uint row, uint col, uint out_dim, uint in_dim, + uint group_size, uint bits) { + uint words_per_row = (in_dim * bits) / 32; + uint groups_per_row = in_dim / group_size; + uint bit_offset = col * bits; + uint word_idx = bit_offset / 32; + uint shift = bit_offset % 32; + uint base = (expert * out_dim + row) * words_per_row; + uint q = w[base + word_idx] >> shift; + if (shift + bits > 32) q |= w[base + word_idx + 1] << (32 - shift); + q &= (1u << bits) - 1u; + uint sb = (expert * out_dim + row) * groups_per_row + col / group_size; + return float(q) * bf16_to_float(s[sb]) + bf16_to_float(b[sb]); + } + kernel void synthetic_moe( + const device float* x [[buffer(0)]], + device float* y [[buffer(1)]], + const device float* scores [[buffer(2)]], + constant FusedParams& p [[buffer(3)]], + uint gid [[thread_position_in_grid]]) { + uint token = gid / p.hidden; + uint col = gid % p.hidden; + float acc = 0.0f; + for (int j = 0; j < p.k; ++j) { + acc += scores[token * p.k + j]; + } + y[gid] = x[token * p.hidden + col] * acc; + } + kernel void fused_moe_pairs( + const device float* x [[buffer(0)]], + const device uint* gate_w [[buffer(1)]], const device ushort* gate_s [[buffer(2)]], const device ushort* gate_b [[buffer(3)]], + const device uint* up_w [[buffer(4)]], const device ushort* up_s [[buffer(5)]], const device ushort* up_b [[buffer(6)]], + const device uint* down_w [[buffer(7)]], const device ushort* down_s [[buffer(8)]], const device ushort* down_b [[buffer(9)]], + device float* y [[buffer(10)]], constant FusedParams& p [[buffer(11)]], + uint tid [[thread_position_in_threadgroup]], uint gid [[thread_position_in_grid]]) { + constexpr uint block_size = 512; + constexpr uint lanes_per_row = 16; + constexpr uint rows_per_step = block_size / lanes_per_row; + uint expert = gid / block_size; + if (expert >= uint(p.experts)) return; + uint token = expert / p.k; + uint local_row = tid / lanes_per_row; + uint row_lane = tid % lanes_per_row; + threadgroup float act[1024]; + threadgroup float gate_part[block_size]; + threadgroup float up_part[block_size]; + for (uint row_base = 0; row_base < uint(p.inter); row_base += rows_per_step) { + uint row = row_base + local_row; + float gate_acc = 0.0f, up_acc = 0.0f; + if (row < uint(p.inter)) { + for (uint col = row_lane; col < uint(p.hidden); col += lanes_per_row) { + float xv = x[token * p.hidden + col]; + gate_acc += qvalue(gate_w, gate_s, gate_b, expert, row, col, p.inter, p.hidden, p.group_size, p.bits) * xv; + up_acc += qvalue(up_w, up_s, up_b, expert, row, col, p.inter, p.hidden, p.group_size, p.bits) * xv; + } + } + gate_part[tid] = gate_acc; + up_part[tid] = up_acc; + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint stride = lanes_per_row / 2; stride > 0; stride >>= 1) { + if (row_lane < stride) { gate_part[tid] += gate_part[tid + stride]; up_part[tid] += up_part[tid + stride]; } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + if (row_lane == 0 && row < uint(p.inter)) { + float g = gate_part[tid]; + act[row] = g * (1.0f / (1.0f + exp(-g))) * up_part[tid]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + for (uint row_base = 0; row_base < uint(p.hidden); row_base += rows_per_step) { + uint row = row_base + local_row; + float acc = 0.0f; + if (row < uint(p.hidden)) { + for (uint col = row_lane; col < uint(p.inter); col += lanes_per_row) { + acc += qvalue(down_w, down_s, down_b, expert, row, col, p.hidden, p.inter, p.group_size, p.bits) * act[col]; + } + } + gate_part[tid] = acc; + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint stride = lanes_per_row / 2; stride > 0; stride >>= 1) { + if (row_lane < stride) gate_part[tid] += gate_part[tid + stride]; + threadgroup_barrier(mem_flags::mem_threadgroup); + } + if (row_lane == 0 && row < uint(p.hidden)) { + y[expert * p.hidden + row] = gate_part[tid]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + } + kernel void fused_moe_slots_pairs( + const device float* x [[buffer(0)]], + const device uint* gate_w [[buffer(1)]], const device ushort* gate_s [[buffer(2)]], const device ushort* gate_b [[buffer(3)]], + const device uint* up_w [[buffer(4)]], const device ushort* up_s [[buffer(5)]], const device ushort* up_b [[buffer(6)]], + const device uint* down_w [[buffer(7)]], const device ushort* down_s [[buffer(8)]], const device ushort* down_b [[buffer(9)]], + device float* y [[buffer(10)]], constant FusedParams& p [[buffer(11)]], + const device uint* local_slots [[buffer(12)]], + uint tid [[thread_position_in_threadgroup]], uint gid [[thread_position_in_grid]]) { + constexpr uint block_size = 512; + constexpr uint lanes_per_row = 16; + constexpr uint rows_per_step = block_size / lanes_per_row; + uint active_idx = gid / block_size; + if (active_idx >= uint(p.experts)) return; + uint slot = local_slots[active_idx]; + uint token = active_idx / p.k; + uint local_row = tid / lanes_per_row; + uint row_lane = tid % lanes_per_row; + threadgroup float act[1024]; + threadgroup float gate_part[block_size]; + threadgroup float up_part[block_size]; + for (uint row_base = 0; row_base < uint(p.inter); row_base += rows_per_step) { + uint row = row_base + local_row; + float gate_acc = 0.0f, up_acc = 0.0f; + if (row < uint(p.inter)) { + for (uint col = row_lane; col < uint(p.hidden); col += lanes_per_row) { + float xv = x[token * p.hidden + col]; + gate_acc += qvalue(gate_w, gate_s, gate_b, slot, row, col, p.inter, p.hidden, p.group_size, p.bits) * xv; + up_acc += qvalue(up_w, up_s, up_b, slot, row, col, p.inter, p.hidden, p.group_size, p.bits) * xv; + } + } + gate_part[tid] = gate_acc; + up_part[tid] = up_acc; + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint stride = lanes_per_row / 2; stride > 0; stride >>= 1) { + if (row_lane < stride) { gate_part[tid] += gate_part[tid + stride]; up_part[tid] += up_part[tid + stride]; } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + if (row_lane == 0 && row < uint(p.inter)) { + float g = gate_part[tid]; + act[row] = g * (1.0f / (1.0f + exp(-g))) * up_part[tid]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + for (uint row_base = 0; row_base < uint(p.hidden); row_base += rows_per_step) { + uint row = row_base + local_row; + float acc = 0.0f; + if (row < uint(p.hidden)) { + for (uint col = row_lane; col < uint(p.inter); col += lanes_per_row) { + acc += qvalue(down_w, down_s, down_b, slot, row, col, p.hidden, p.inter, p.group_size, p.bits) * act[col]; + } + } + gate_part[tid] = acc; + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint stride = lanes_per_row / 2; stride > 0; stride >>= 1) { + if (row_lane < stride) gate_part[tid] += gate_part[tid + stride]; + threadgroup_barrier(mem_flags::mem_threadgroup); + } + if (row_lane == 0 && row < uint(p.hidden)) { + y[active_idx * p.hidden + row] = gate_part[tid]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + } + kernel void reduce_pairs( + const device float* pair_y [[buffer(0)]], + const device float* scores [[buffer(1)]], + device float* out [[buffer(2)]], + constant FusedParams& p [[buffer(3)]], + uint gid [[thread_position_in_grid]]) { + uint token = gid / p.hidden; + uint col = gid % p.hidden; + float acc = 0.0f; + for (int j = 0; j < p.k; ++j) { + uint pair = token * p.k + uint(j); + acc += pair_y[pair * p.hidden + col] * scores[pair]; + } + out[gid] = acc; + } + )"; +} + +class FusedMoePrimitive : public mx::Primitive { + public: + FusedMoePrimitive( + mx::Stream stream, + std::string compute_dir, + int layer, + int hidden, + int inter, + int group, + int bits, + int num_experts, + bool synthetic, + std::vector expert_ids) + : Primitive(stream), + compute_dir_(std::move(compute_dir)), + layer_(layer), + hidden_(hidden), + inter_(inter), + group_(group), + bits_(bits), + num_experts_(num_experts), + synthetic_(synthetic), + expert_ids_(std::move(expert_ids)) { + if (!synthetic_) { + gate_ = map_proj(compute_dir_, layer_, "gate_proj"); + up_ = map_proj(compute_dir_, layer_, "up_proj"); + down_ = map_proj(compute_dir_, layer_, "down_proj"); + } + } + + ~FusedMoePrimitive() override { + if (!synthetic_) { + unmap_proj(gate_); + unmap_proj(up_); + unmap_proj(down_); + } + } + + const char* name() const override { return "FusedMoePrimitive"; } + + void eval_cpu(const std::vector&, std::vector&) override { + throw std::runtime_error("FusedMoePrimitive only supports GPU evaluation"); + } + + void eval_gpu(const std::vector& inputs, std::vector& outputs) override { + auto& x = inputs[0]; + auto& scores = inputs[1]; + auto& out = outputs[0]; + int active = static_cast(expert_ids_.size()); + int tokens = static_cast(x.size() / hidden_); + int k = active / std::max(1, tokens); + if (active <= 0 || tokens <= 0 || active != static_cast(scores.size())) { + throw std::runtime_error("FusedMoePrimitive shape mismatch"); + } + out.set_data(mx::allocator::malloc(out.nbytes())); + auto& d = mx::metal::device(stream().device); + auto& enc = mx::metal::get_command_encoder(stream()); + auto lib = d.get_library("native_moe_mlx_ext", []() { return metal_source(); }); + if (synthetic_) { + auto synthetic_kernel = d.get_kernel("synthetic_moe", lib); + enc.set_compute_pipeline_state(synthetic_kernel); + enc.set_input_array(x, 0); + enc.set_output_array(out, 1); + enc.set_input_array(scores, 2); + FusedParams params{active, hidden_, inter_, group_, bits_, k}; + enc.set_bytes(params, 3); + enc.dispatch_threads(MTL::Size(out.size(), 1, 1), MTL::Size(std::min(out.size(), 256), 1, 1)); + return; + } + auto fused = d.get_kernel("fused_moe_pairs", lib); + auto reduce = d.get_kernel("reduce_pairs", lib); + + ProjectionShape gu = proj_shape(inter_, hidden_, group_, bits_); + ProjectionShape down = proj_shape(hidden_, inter_, group_, bits_); + size_t gu_w_bytes = static_cast(active) * gu.weight_per; + size_t gu_s_bytes = static_cast(active) * gu.scale_per; + size_t down_w_bytes = static_cast(active) * down.weight_per; + size_t down_s_bytes = static_cast(active) * down.scale_per; + buffers_.clear(); + auto make_buffer = [&](size_t nbytes) -> MTL::Buffer* { + buffers_.push_back(NS::TransferPtr(d.mtl_device()->newBuffer(nbytes, MTL::ResourceStorageModeShared))); + return buffers_.back().get(); + }; + MTL::Buffer* gw = make_buffer(gu_w_bytes); + MTL::Buffer* gs = make_buffer(gu_s_bytes); + MTL::Buffer* gb = make_buffer(gu_s_bytes); + MTL::Buffer* uw = make_buffer(gu_w_bytes); + MTL::Buffer* us = make_buffer(gu_s_bytes); + MTL::Buffer* ub = make_buffer(gu_s_bytes); + MTL::Buffer* dw = make_buffer(down_w_bytes); + MTL::Buffer* ds = make_buffer(down_s_bytes); + MTL::Buffer* db = make_buffer(down_s_bytes); + if (synthetic_) { + fill_synthetic(gw, gs, gb, 1); + fill_synthetic(uw, us, ub, 2); + fill_synthetic(dw, ds, db, 3); + } else { + copy_projection(gate_, gu, expert_ids_, gw, gs, gb); + copy_projection(up_, gu, expert_ids_, uw, us, ub); + copy_projection(down_, down, expert_ids_, dw, ds, db); + } + + auto pair_out = mx::zeros(mx::Shape{active, hidden_}, mx::float32, stream()); + pair_out.set_data(mx::allocator::malloc(pair_out.nbytes())); + + enc.set_compute_pipeline_state(fused); + enc.set_input_array(x, 0); + enc.set_buffer(gw, 1); + enc.set_buffer(gs, 2); + enc.set_buffer(gb, 3); + enc.set_buffer(uw, 4); + enc.set_buffer(us, 5); + enc.set_buffer(ub, 6); + enc.set_buffer(dw, 7); + enc.set_buffer(ds, 8); + enc.set_buffer(db, 9); + enc.set_output_array(out, 10); + FusedParams params{active, hidden_, inter_, group_, bits_, k}; + enc.set_bytes(params, 11); + enc.set_input_array(scores, 12); + enc.add_temporary(x); + enc.add_temporary(scores); + enc.dispatch_threads(MTL::Size(static_cast(active) * 512, 1, 1), MTL::Size(512, 1, 1)); + } + + private: + std::string compute_dir_; + int layer_; + int hidden_; + int inter_; + int group_; + int bits_; + int num_experts_; + bool synthetic_; + std::vector expert_ids_; + MappedProj gate_; + MappedProj up_; + MappedProj down_; + std::vector> buffers_; +}; + +class StagedFusedMoePrimitive : public mx::Primitive { + public: + StagedFusedMoePrimitive(mx::Stream stream, int hidden, int inter, int group, int bits) + : Primitive(stream), hidden_(hidden), inter_(inter), group_(group), bits_(bits) {} + + const char* name() const override { return "StagedFusedMoePrimitive"; } + + void eval_cpu(const std::vector&, std::vector&) override { + throw std::runtime_error("StagedFusedMoePrimitive only supports GPU evaluation"); + } + + void eval_gpu(const std::vector& inputs, std::vector& outputs) override { + auto& x = inputs[0]; + auto& scores = inputs[1]; + auto& out = outputs[0]; + int tokens = static_cast(x.size() / hidden_); + int active = static_cast(scores.size()); + int k = active / std::max(1, tokens); + if (active <= 0 || tokens <= 0 || active != static_cast(scores.size())) { + throw std::runtime_error("StagedFusedMoePrimitive shape mismatch"); + } + out.set_data(mx::allocator::malloc(out.nbytes())); + auto& d = mx::metal::device(stream().device); + auto& enc = mx::metal::get_command_encoder(stream()); + auto lib = d.get_library("native_moe_mlx_ext", []() { return metal_source(); }); + auto fused = d.get_kernel("fused_moe_pairs", lib); + auto reduce = d.get_kernel("reduce_pairs", lib); + auto pair_out = mx::zeros(mx::Shape{active, hidden_}, mx::float32, stream()); + pair_out.set_data(mx::allocator::malloc(pair_out.nbytes())); + enc.set_compute_pipeline_state(fused); + enc.set_input_array(x, 0); + enc.set_input_array(inputs[2], 1); + enc.set_input_array(inputs[3], 2); + enc.set_input_array(inputs[4], 3); + enc.set_input_array(inputs[5], 4); + enc.set_input_array(inputs[6], 5); + enc.set_input_array(inputs[7], 6); + enc.set_input_array(inputs[8], 7); + enc.set_input_array(inputs[9], 8); + enc.set_input_array(inputs[10], 9); + enc.set_output_array(pair_out, 10); + FusedParams params{active, hidden_, inter_, group_, bits_, k}; + enc.set_bytes(params, 11); + enc.dispatch_threads(MTL::Size(static_cast(active) * 512, 1, 1), MTL::Size(512, 1, 1)); + enc.set_compute_pipeline_state(reduce); + enc.set_input_array(pair_out, 0); + enc.set_input_array(scores, 1); + enc.set_output_array(out, 2); + enc.set_bytes(params, 3); + enc.add_temporary(pair_out); + enc.dispatch_threads(MTL::Size(out.size(), 1, 1), MTL::Size(std::min(out.size(), 256), 1, 1)); + } + + private: + int hidden_; + int inter_; + int group_; + int bits_; +}; + +class SlotFusedMoePrimitive : public mx::Primitive { + public: + SlotFusedMoePrimitive(mx::Stream stream, int hidden, int inter, int group, int bits) + : Primitive(stream), hidden_(hidden), inter_(inter), group_(group), bits_(bits) {} + + const char* name() const override { return "SlotFusedMoePrimitive"; } + + void eval_cpu(const std::vector&, std::vector&) override { + throw std::runtime_error("SlotFusedMoePrimitive only supports GPU evaluation"); + } + + void eval_gpu(const std::vector& inputs, std::vector& outputs) override { + auto& x = inputs[0]; + auto& local_slots = inputs[1]; + auto& scores = inputs[2]; + auto& out = outputs[0]; + int tokens = static_cast(x.size() / hidden_); + int active = static_cast(scores.size()); + int k = active / std::max(1, tokens); + if (active <= 0 || tokens <= 0 || active != static_cast(local_slots.size())) { + throw std::runtime_error("SlotFusedMoePrimitive shape mismatch"); + } + out.set_data(mx::allocator::malloc(out.nbytes())); + auto& d = mx::metal::device(stream().device); + auto& enc = mx::metal::get_command_encoder(stream()); + auto lib = d.get_library("native_moe_mlx_ext", []() { return metal_source(); }); + auto fused = d.get_kernel("fused_moe_slots_pairs", lib); + auto reduce = d.get_kernel("reduce_pairs", lib); + auto pair_out = mx::zeros(mx::Shape{active, hidden_}, mx::float32, stream()); + pair_out.set_data(mx::allocator::malloc(pair_out.nbytes())); + enc.set_compute_pipeline_state(fused); + enc.set_input_array(x, 0); + enc.set_input_array(inputs[3], 1); + enc.set_input_array(inputs[4], 2); + enc.set_input_array(inputs[5], 3); + enc.set_input_array(inputs[6], 4); + enc.set_input_array(inputs[7], 5); + enc.set_input_array(inputs[8], 6); + enc.set_input_array(inputs[9], 7); + enc.set_input_array(inputs[10], 8); + enc.set_input_array(inputs[11], 9); + enc.set_output_array(pair_out, 10); + FusedParams params{active, hidden_, inter_, group_, bits_, k}; + enc.set_bytes(params, 11); + enc.set_input_array(local_slots, 12); + enc.dispatch_threads(MTL::Size(static_cast(active) * 512, 1, 1), MTL::Size(512, 1, 1)); + enc.set_compute_pipeline_state(reduce); + enc.set_input_array(pair_out, 0); + enc.set_input_array(scores, 1); + enc.set_output_array(out, 2); + enc.set_bytes(params, 3); + enc.add_temporary(pair_out); + enc.dispatch_threads(MTL::Size(out.size(), 1, 1), MTL::Size(std::min(out.size(), 256), 1, 1)); + } + + private: + int hidden_; + int inter_; + int group_; + int bits_; +}; +mx::array fused_moe( + const mx::array& x, + const mx::array& expert_ids, + const mx::array& scores, + const std::string& compute_dir, + int layer, + int hidden, + int inter, + int group, + int bits, + int num_experts, + bool synthetic, + mx::StreamOrDevice s = {}) { + if (!synthetic) { + throw std::runtime_error( + "real mmap staging in MLX Primitive needs MLX-managed staging buffers; use synthetic for bridge tests"); + } + auto ids = expert_ids; + ids.eval(); + std::vector expert_vec(ids.size()); + const uint32_t* idp = ids.data(); + for (size_t i = 0; i < ids.size(); ++i) expert_vec[i] = static_cast(idp[i]); + auto out_shape = x.shape(); + out_shape.back() = hidden; + return mx::array( + out_shape, + mx::float32, + std::make_shared( + mx::to_stream(s), + compute_dir, + layer, + hidden, + inter, + group, + bits, + num_experts, + synthetic, + std::move(expert_vec)), + std::vector{x, scores}); +} + +mx::array fused_moe_staged( + const mx::array& x, + const mx::array& scores, + const mx::array& gate_w, + const mx::array& gate_s, + const mx::array& gate_b, + const mx::array& up_w, + const mx::array& up_s, + const mx::array& up_b, + const mx::array& down_w, + const mx::array& down_s, + const mx::array& down_b, + int hidden, + int inter, + int group, + int bits, + mx::StreamOrDevice s = {}) { + auto out_shape = x.shape(); + out_shape.back() = hidden; + return mx::array( + out_shape, + mx::float32, + std::make_shared(mx::to_stream(s), hidden, inter, group, bits), + std::vector{ + x, scores, + gate_w, gate_s, gate_b, + up_w, up_s, up_b, + down_w, down_s, down_b}); +} + +mx::array fused_moe_slots( + const mx::array& x, + const mx::array& local_slots, + const mx::array& scores, + const mx::array& gate_w, + const mx::array& gate_s, + const mx::array& gate_b, + const mx::array& up_w, + const mx::array& up_s, + const mx::array& up_b, + const mx::array& down_w, + const mx::array& down_s, + const mx::array& down_b, + int hidden, + int inter, + int group, + int bits, + mx::StreamOrDevice s = {}) { + auto out_shape = x.shape(); + out_shape.back() = hidden; + return mx::array( + out_shape, + mx::float32, + std::make_shared(mx::to_stream(s), hidden, inter, group, bits), + std::vector{ + x, local_slots, scores, + gate_w, gate_s, gate_b, + up_w, up_s, up_b, + down_w, down_s, down_b}); +} diff --git a/native/ext/compute/fused_moe.h b/native/ext/compute/fused_moe.h new file mode 100644 index 0000000..23aac04 --- /dev/null +++ b/native/ext/compute/fused_moe.h @@ -0,0 +1,23 @@ +// fused MoE 计算：合成/真实 mmap staging、staged(预切片权重)、slot(local slot 索引)三条工厂。 +// 实现见 fused_moe.cpp；这里只暴露给 bindings 的自由函数声明（默认参数放在声明处）。 +#pragma once +#include "../common.h" + +mx::array fused_moe( + const mx::array& x, const mx::array& expert_ids, const mx::array& scores, + const std::string& compute_dir, int layer, int hidden, int inter, int group, + int bits, int num_experts, bool synthetic, mx::StreamOrDevice s); + +mx::array fused_moe_staged( + const mx::array& x, const mx::array& scores, + const mx::array& gate_w, const mx::array& gate_s, const mx::array& gate_b, + const mx::array& up_w, const mx::array& up_s, const mx::array& up_b, + const mx::array& down_w, const mx::array& down_s, const mx::array& down_b, + int hidden, int inter, int group, int bits, mx::StreamOrDevice s); + +mx::array fused_moe_slots( + const mx::array& x, const mx::array& local_slots, const mx::array& scores, + const mx::array& gate_w, const mx::array& gate_s, const mx::array& gate_b, + const mx::array& up_w, const mx::array& up_s, const mx::array& up_b, + const mx::array& down_w, const mx::array& down_s, const mx::array& down_b, + int hidden, int inter, int group, int bits, mx::StreamOrDevice s); diff --git a/native/ext/io/bg_reader.cpp b/native/ext/io/bg_reader.cpp new file mode 100644 index 0000000..9969656 --- /dev/null +++ b/native/ext/io/bg_reader.cpp @@ -0,0 +1,183 @@ +// [7] 自由后台读线程（de-risk）：双队列 + 低优并发上限的后台 pread 线程池。 +#include "bg_reader.h" +#include "blob_io.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +struct ReadOp { uint8_t* dst; size_t nbytes; off_t file_off; }; +struct BgJob { + std::vector ops; + std::string path; + long ticket; + std::vector keep; // 持 buffer 引用，保证 dst 指针在读期间存活 + std::function task; // 若设置，worker 直接执行它（侧区异步读用），不走 ops/ticket + int prio = 0; // >0=高优(route 读)，=0=低优(投机兜底) + bool nocache = true; // demand（route）读设 false 走 page cache：段偏移非页对齐， + // F_NOCACHE 下多 MB 非对齐 pread 会短读 → 池槽脏字节（实测）。 +}; + +// 双队列 + 低优并发上限：高优(route)读永远优先取、且独占大多数 worker； +// 低优(投机)读最多占 low_cap_ 个 worker → 给 route 读留出 worker 和 SSD 带宽， +// 实现"高分抢带宽先就绪、低分延后不阻塞关键路径"的真 IO 优先级。 +class BgReader { + public: + ~BgReader() { stop(); } // 进程退出兜底：避免 joinable 线程析构触发 std::terminate + void start(int workers, int low_cap) { + { std::lock_guard lk(m_); low_cap_ = low_cap; } + ensure_started(workers); + } + // 懒启动：侧区预取的 GPU 回调可能在 Python 没调用 bg_reader_start 时就要派任务， + // 故 submit/submit_task 自动确保线程已起（默认 4 worker），调用方无需显式 start。 + void ensure_started(int workers) { + std::lock_guard lk(m_); + if (running_) return; + running_ = true; + for (int i = 0; i < workers; ++i) threads_.emplace_back([this] { loop(); }); + } + void submit(BgJob job) { + ensure_started(4); + { std::lock_guard lk(m_); + if (job.prio > 0) high_q_.push(std::move(job)); else low_q_.push(std::move(job)); } + cv_.notify_all(); + } + void submit_task(std::function fn) { + ensure_started(4); + BgJob job; + job.task = std::move(fn); + { std::lock_guard lk(m_); low_q_.push(std::move(job)); } // 通用任务走低优 + cv_.notify_all(); + } + bool ready(long ticket) { + std::lock_guard lk(dm_); + return done_.count(ticket) > 0; + } + void wait(long ticket) { + std::unique_lock lk(dm_); + dcv_.wait(lk, [&] { return done_.count(ticket) > 0; }); + } + void stop() { + { std::lock_guard lk(m_); running_ = false; } + cv_.notify_all(); + for (auto& t : threads_) if (t.joinable()) t.join(); + threads_.clear(); + { std::lock_guard lk(dm_); done_.clear(); } + { std::lock_guard lk(m_); active_low_ = 0; } + } + private: + int low_budget() const { return low_cap_ <= 0 ? 1 << 30 : low_cap_; } // <=0 视为不限流 + bool can_take_low() const { return !low_q_.empty() && active_low_ < low_budget(); } + void loop() { + std::unordered_map fds; // 每线程各自缓存 fd + while (true) { + std::unique_lock lk(m_); + cv_.wait(lk, [this] { return !high_q_.empty() || can_take_low() || !running_; }); + if (!running_ && high_q_.empty() && low_q_.empty()) break; + if (!running_ && high_q_.empty() && !can_take_low()) break; // 退出时低优可超额排空 + bool is_low = false; + BgJob job; + if (!high_q_.empty()) { // 高优(route)读永远先取 + job = std::move(high_q_.front()); high_q_.pop(); + } else if (can_take_low()) { // 低优限流：仅在额度内取 + job = std::move(low_q_.front()); low_q_.pop(); ++active_low_; is_low = true; + } else { + continue; // 假醒（低优已满额）→ 回去等 + } + lk.unlock(); + if (job.task) { job.task(); } // 通用任务（侧区异步读）：直接执行，无 ticket + else { + int fd; + std::string key = (job.nocache ? "N:" : "C:") + job.path; // 按 nocache 分别缓存 fd + auto it = fds.find(key); + if (it == fds.end()) { + fd = job.nocache ? open_blob_nocache(job.path.c_str()) : ::open(job.path.c_str(), O_RDONLY); + fds[key] = fd; + } else fd = it->second; + if (fd >= 0) + for (auto& op : job.ops) { + ssize_t got = ::pread(fd, op.dst, op.nbytes, op.file_off); + if (got != static_cast(op.nbytes)) + fprintf(stderr, "[bg pread SHORT] got=%zd want=%zu off=%lld\n", + got, op.nbytes, static_cast(op.file_off)); + } + { std::lock_guard lk2(dm_); done_.insert(job.ticket); } + dcv_.notify_all(); + } + if (is_low) { // 释放低优额度 → 唤醒别的 worker 再取 + std::lock_guard lk2(m_); --active_low_; cv_.notify_all(); + } + } + for (auto& kv : fds) if (kv.second >= 0) ::close(kv.second); + } + std::mutex m_, dm_; + std::condition_variable cv_, dcv_; + std::queue high_q_, low_q_; + int active_low_ = 0; // 当前正在执行的低优读数 + int low_cap_ = 0; // 低优并发上限（<=0 不限流，保持旧行为） + std::unordered_set done_; + std::vector threads_; + bool running_ = false; +}; +BgReader g_bg; +} // namespace + +void bg_submit_task(std::function fn) { g_bg.submit_task(std::move(fn)); } + +void bg_reader_start(int workers, int low_cap) { g_bg.start(workers, low_cap); } + +long bg_reader_submit(const mx::array& dst, const std::vector& experts, + const std::vector& rows, const std::string& path, + int stride, long ticket, int prio) { + mx::array d = dst; + d.eval(); + uint8_t* base = d.data(); + size_t st = static_cast(stride); + BgJob job; + job.path = path; + job.ticket = ticket; + job.prio = prio; + job.keep.push_back(d); + for (size_t i = 0; i < experts.size(); ++i) + job.ops.push_back(ReadOp{base + static_cast(rows[i]) * st, st, + static_cast(static_cast(experts[i]) * st)}); + g_bg.submit(std::move(job)); + return ticket; +} + +// 专家各段直写进多个池段张量的 slot 行（消费侧零 MLX 算子）。 +long bg_pread_into_pool( + const std::vector& dst, + const std::vector& seg_off, + const std::vector& seg_nb, + long slot, long expert, + const std::string& path, long stride, long ticket, int prio, bool nocache) { + BgJob job; + job.path = path; + job.ticket = ticket; + job.prio = prio; + job.nocache = nocache; + for (size_t i = 0; i < dst.size(); ++i) { + mx::array d = dst[i]; + d.eval(); + uint8_t* base = d.data(); + job.keep.push_back(d); + job.ops.push_back(ReadOp{ + base + static_cast(slot) * static_cast(seg_nb[i]), + static_cast(seg_nb[i]), + static_cast(static_cast(expert) * static_cast(stride) + + static_cast(seg_off[i]))}); + } + g_bg.submit(std::move(job)); + return ticket; +} + +bool bg_reader_ready(long ticket) { return g_bg.ready(ticket); } +void bg_reader_wait(long ticket) { g_bg.wait(ticket); } +void bg_reader_stop() { g_bg.stop(); } diff --git a/native/ext/io/bg_reader.h b/native/ext/io/bg_reader.h new file mode 100644 index 0000000..779ce87 --- /dev/null +++ b/native/ext/io/bg_reader.h @@ -0,0 +1,23 @@ +// 自由后台读线程（de-risk）：脱离 GPU 完成回调、零 GIL，pread 进调用方 MLX buffer。 +// 线程全程只碰 dst 原始指针 / 路径 / 整数，绝不接触 Python 对象 → 无需 GIL。 +#pragma once +#include "../common.h" +#include + +void bg_reader_start(int workers, int low_cap = 0); +long bg_reader_submit(const mx::array& dst, const std::vector& experts, + const std::vector& rows, const std::string& path, + int stride, long ticket, int prio = 0); +bool bg_reader_ready(long ticket); +void bg_reader_wait(long ticket); +void bg_reader_stop(); +long bg_pread_into_pool( + const std::vector& dst, + const std::vector& seg_off, + const std::vector& seg_nb, + long slot, long expert, + const std::string& path, long stride, long ticket, int prio = 0, bool nocache = true); + +// 通用后台任务入口：把任意闭包派到后台线程池（低优队列）。侧区/staging 预取的 GPU 完成回调 +// 用它把 pread 派离 Metal 回调线程 → 真正与计算重叠。内部接口，不绑 Python。 +void bg_submit_task(std::function fn); diff --git a/native/ext/io/blob_io.h b/native/ext/io/blob_io.h new file mode 100644 index 0000000..793b496 --- /dev/null +++ b/native/ext/io/blob_io.h @@ -0,0 +1,16 @@ +// 共用底层 IO 助手：打开 blob 文件。多个模块（侧区/后台读线程等）的 pread 都用它。 +#pragma once +#include +#include + +#ifndef F_NOCACHE +#define F_NOCACHE 48 // macOS：提示内核读过的页不留 page cache +#endif + +// 打开 blob 并设 F_NOCACHE：与 demand 侧 blob_loader 对齐，避免预取读把 page cache 灌满 → +// 在内存受限机上累积压力触发"双稳态"慢挡翻转（实测 zerocopy 每轮翻慢挡的根因）。 +static inline int open_blob_nocache(const char* path) { + int fd = ::open(path, O_RDONLY); + if (fd >= 0) ::fcntl(fd, F_NOCACHE, 1); + return fd; +} diff --git a/native/ext/io/blob_load.cpp b/native/ext/io/blob_load.cpp new file mode 100644 index 0000000..efbd389 --- /dev/null +++ b/native/ext/io/blob_load.cpp @@ -0,0 +1,58 @@ +// [1] blob 直读：pread 专家字节进 MLX buffer（惰性图节点）。 +// 把一组专家的 blob 字节直接 pread 进 MLX 自有 buffer（无 kernel、无额外拷贝）。 +// load 作为惰性图节点：在批量 eval 中执行，避免 Python 侧 per-expert mx.eval 同步。 +#include "blob_load.h" + +#include +#include + +class BlobLoadPrimitive : public mx::Primitive { + public: + BlobLoadPrimitive(mx::Stream stream, std::string path, size_t stride, std::vector experts) + : Primitive(stream), path_(std::move(path)), stride_(stride), experts_(std::move(experts)) {} + const char* name() const override { return "BlobLoadPrimitive"; } + void eval_cpu(const std::vector&, std::vector& outputs) override { + load(outputs[0]); + } + void eval_gpu(const std::vector&, std::vector& outputs) override { + load(outputs[0]); + } + + private: + void load(mx::array& out) { + out.set_data(mx::allocator::malloc(out.nbytes())); + uint8_t* dst = out.data(); + int fd = ::open(path_.c_str(), O_RDONLY); + if (fd < 0) throw std::runtime_error("blob open failed: " + path_); + for (size_t i = 0; i < experts_.size(); ++i) { + size_t off = static_cast(experts_[i]) * stride_; + ssize_t n = ::pread(fd, dst + i * stride_, stride_, static_cast(off)); + if (n != static_cast(stride_)) { + ::close(fd); + throw std::runtime_error("blob pread short read"); + } + } + ::close(fd); + } + std::string path_; + size_t stride_; + std::vector experts_; +}; + +mx::array blob_load( + const std::string& path, + const mx::array& expert_ids, + int stride, + mx::StreamOrDevice s = {}) { + auto ids = expert_ids; + ids.eval(); + std::vector ev(ids.size()); + const uint32_t* p = ids.data(); + for (size_t i = 0; i < ids.size(); ++i) ev[i] = static_cast(p[i]); + int n = static_cast(ev.size()); + return mx::array( + mx::Shape{n, stride}, + mx::uint8, + std::make_shared(mx::to_stream(s), path, static_cast(stride), std::move(ev)), + std::vector{}); +} diff --git a/native/ext/io/blob_load.h b/native/ext/io/blob_load.h new file mode 100644 index 0000000..808dfed --- /dev/null +++ b/native/ext/io/blob_load.h @@ -0,0 +1,7 @@ +// blob 直读：把一组专家 blob 字节 pread 进新建的 MLX uint8[n,stride] 数组（惰性图节点）。 +#pragma once +#include "../common.h" + +mx::array blob_load( + const std::string& path, const mx::array& expert_ids, int stride, + mx::StreamOrDevice s); diff --git a/native/ext/pool/demand.cpp b/native/ext/pool/demand.cpp new file mode 100644 index 0000000..bbe53c1 --- /dev/null +++ b/native/ext/pool/demand.cpp @@ -0,0 +1,331 @@ +// [6] Phase 2 方案B：真实区槽状态 C++ 全接管（1 次同步版）。 +// 复刻 Python ResidentExpertPool 的 _slot_of/_free/_freq + _choose_victim/_alloc_slot 语义： +// - free 优先(front pop，free 初始 [0,cap))；free 空 → LFU 驱逐，受害者槽直接复用(不回 free)。 +// - _choose_victim：candidates=插入序中 ∉current 者；victim=min(freq, 候选下标)，驱逐不删 freq。 +// - dual 语义：真实区命中不移动插入序；仅新放入 miss 追加序尾。 +#include "demand.h" +#include "side_region.h" +#include "../io/bg_reader.h" + +#include +#include +#include +#include +#include + +struct RealLayer { + std::vector order; // 插入序(LRU tie-break)，与 e2r 同步维护 + std::unordered_map e2r; // expert -> slot [0,cap) + std::vector free_rows; // 空闲槽(front pop，仿 free.pop(0)) + std::unordered_map freq; // LFU 频次(驱逐不删，与 Python 一致) + std::unordered_set pinned; // AUTOPIN 钉死集：驱逐永不选(见 real_pin) + int cap = 0; + long access = 0; // 累计访问(decay 用) + bool inited = false; +}; +static std::mutex g_real_mutex; +static std::map g_real; + +// demand 统计：累计 + 本次(供 Python 更新 rp.hits/misses/gpu_fastpath/gpu_fallback)。 +static std::mutex g_dstat_mutex; +// 本次 [hitpos, misspos, loads, fallback01, unplaced] +static long g_d_last[5] = {0, 0, 0, 0, 0}; +static std::atomic g_demand_ticket{1000000000}; // demand 并行 pread 用的独立 ticket 段 + +// 诊断计时(DEMAND_TIMING=1)：累计各段主线程微秒，供定位结构性开销。默认关。 +static bool g_dt_on = false; +static double g_dt[6] = {0, 0, 0, 0, 0, 0}; // [inds_eval, pool_eval, side_snap, real_lock, core, build] +static inline double dt_now_us() { + return std::chrono::duration( + std::chrono::steady_clock::now().time_since_epoch()).count(); +} +std::vector demand_timings() { return {g_dt[0], g_dt[1], g_dt[2], g_dt[3], g_dt[4], g_dt[5]}; } +void demand_timing_enable(bool on) { + g_dt_on = on; + for (int i = 0; i < 6; ++i) g_dt[i] = 0; +} + +static void real_ensure_locked(RealLayer& c, int cap) { + if (c.inited) return; + c.cap = cap; + c.free_rows.clear(); + for (int r = 0; r < cap; ++r) c.free_rows.push_back(r); // free 初始 [0,cap) + c.inited = true; +} + +void real_init(int layer, int cap) { + std::lock_guard lk(g_real_mutex); + real_ensure_locked(g_real[layer], cap); +} + +std::vector real_region_contents(int layer) { + std::lock_guard lk(g_real_mutex); + std::vector out; + auto it = g_real.find(layer); + if (it != g_real.end()) + for (auto& p : it->second.e2r) { out.push_back(p.first); out.push_back(p.second); } + return out; +} + +int real_region_count(int layer) { + std::lock_guard lk(g_real_mutex); + auto it = g_real.find(layer); + return it == g_real.end() ? 0 : static_cast(it->second.e2r.size()); +} + +int real_pinned_count(int layer) { + std::lock_guard lk(g_real_mutex); + auto it = g_real.find(layer); + return it == g_real.end() ? 0 : static_cast(it->second.pinned.size()); +} + +void real_reset() { + std::lock_guard lk(g_real_mutex); + g_real.clear(); +} + +// 复刻 _choose_victim：遍历插入序(order)选 ∉current 且 freq 最小者，并列取最早(候选下标最小)。 +// pinned(AUTOPIN 钉死)与 current 同样永不选。返回 expert id；-1 表示无可驱逐。调用方须持 g_real_mutex。 +static int choose_victim_locked(RealLayer& c, const std::unordered_set& current) { + int victim = -1; + uint32_t best = 0; + for (int e : c.order) { + if (!c.e2r.count(e) || current.count(e) || c.pinned.count(e)) continue; + uint32_t f = c.freq.count(e) ? c.freq[e] : 0; + if (victim < 0 || f < best) { victim = e; best = f; } // 并列不更新 → 保留更早者 + } + return victim; +} + +// AUTOPIN 预热钉死：把历史热专家注册进真实区并标记 pinned(此后驱逐永不选它)。 +// 已驻 → 仅补 pinned 标记、返回原槽；未驻 → free 头取槽登记(池行字节由调用方负责写入)。 +// 返回与 experts 平行的槽位；空闲耗尽分不到槽给 -1(调用方跳过该专家)。幂等。 +// 前置：调用方须先 real_init(否则 free_rows 为空、全返 -1)。 +std::vector real_pin(int layer, const std::vector& experts) { + std::lock_guard lk(g_real_mutex); + RealLayer& c = g_real[layer]; + std::vector out; + out.reserve(experts.size()); + for (int e : experts) { + auto it = c.e2r.find(e); + if (it != c.e2r.end()) { c.pinned.insert(e); out.push_back(it->second); continue; } + if (c.free_rows.empty()) { out.push_back(-1); continue; } + int slot = c.free_rows.front(); + c.free_rows.erase(c.free_rows.begin()); // free.pop(0) + c.e2r[e] = slot; + c.order.push_back(e); + c.pinned.insert(e); + out.push_back(slot); + } + return out; +} + +// AUTOPIN 热度持久化：导出各层 LFU 累计频次，扁平 [layer, expert, count, ...]。 +// 仅 lfu 策略下 demand 路径计频；驱逐不删 freq，故即该进程全生命周期的路由热度， +// 由 Python 侧按增量差分合并进 usage 文件(dual decode 无 Python 物化点，这是唯一计频来源)。 +std::vector real_freq_dump() { + std::lock_guard lk(g_real_mutex); + std::vector out; + for (auto& kv : g_real) + for (auto& p : kv.second.freq) { + out.push_back(static_cast(kv.first)); + out.push_back(static_cast(p.first)); + out.push_back(static_cast(p.second)); + } + return out; +} + +// 复刻 _alloc_slot（e 为 miss，不会已在 e2r）：free 优先，否则 LFU 驱逐复用受害者槽。 +// 返回 slot；-1 表示无可驱逐(超容量，不该在 dual 发生)。调用方须持 g_real_mutex。 +static int alloc_slot_locked(RealLayer& c, int e, const std::unordered_set& current) { + auto it = c.e2r.find(e); + if (it != c.e2r.end()) return it->second; + int slot; + if (!c.free_rows.empty()) { + slot = c.free_rows.front(); + c.free_rows.erase(c.free_rows.begin()); // free.pop(0) + } else { + int victim = choose_victim_locked(c, current); + if (victim < 0) return -1; + slot = c.e2r[victim]; + c.e2r.erase(victim); + for (auto oit = c.order.begin(); oit != c.order.end(); ++oit) + if (*oit == victim) { c.order.erase(oit); break; } + } + c.e2r[e] = slot; + c.order.push_back(e); + return slot; +} + +// LFU 频次 bump（canonical：本次全部唯一专家各 +1）+ decay。调用方须持 g_real_mutex。 +static void note_access_locked(RealLayer& c, const std::vector& uniq_access, + bool lfu, int decay_interval) { + if (!lfu) return; + for (int e : uniq_access) c.freq[e] += 1; + c.access += static_cast(uniq_access.size()); + if (decay_interval > 0 && c.access >= decay_interval) { + for (auto it = c.freq.begin(); it != c.freq.end();) { + it->second /= 2; + if (it->second == 0) it = c.freq.erase(it); + else ++it; + } + c.access = 0; + } +} + +// 核心状态机（纯 CPU、无 I/O）：给定 host inds(ip[n]) + 侧区快照(side)，算 local、分配 miss 槽， +// 把需要落盘的新放入 (expert, slot) 追加到 placements（字节由调用方在锁外并行 pread 落池）。 +// 返回 local(int32 vector)。stats: [hitpos, misspos, loads(=placements 数), unplaced]。 +static std::vector demand_core_locked( + RealLayer& c, const uint32_t* ip, size_t n, const std::unordered_map& side, + bool lfu, int decay_interval, std::vector>& placements, long stats[4]) { + std::vector local(n, -1); + std::vector uniq_miss, access_order; + std::unordered_set miss_seen, access_seen; + int hitpos = 0; + // pass1：算命中(侧区覆盖真实区)、收集 miss(首见序)、收集唯一访问(freq)。 + for (size_t i = 0; i < n; ++i) { + int e = static_cast(ip[i]); + if (access_seen.insert(e).second) access_order.push_back(e); + auto sit = side.find(e); + if (sit != side.end()) { local[i] = sit->second; ++hitpos; continue; } + auto rit = c.e2r.find(e); + if (rit != c.e2r.end()) { local[i] = rit->second; ++hitpos; continue; } + if (miss_seen.insert(e).second) uniq_miss.push_back(e); + } + note_access_locked(c, access_order, lfu, decay_interval); + // pass2：miss 分配槽（不落盘）。current = 本前向全部唯一路由专家(命中+miss)：绝不驱逐本前向要读的 + // 任何专家的槽（否则真实区命中专家的槽被 miss 复写 → 脏字节。比 Python 仅护 miss 更严格、更正确）。 + std::unordered_map new_slot; + long unplaced = 0; + for (int e : uniq_miss) { + int slot = alloc_slot_locked(c, e, access_seen); + if (slot < 0) { + // 超容量：可驱逐槽(= cap − pinned − 本前向命中数)不够安放全部 miss。只能落 0 号槽， + // 即拿别的专家的权重参与计算 → 该前向逐位错算。调用方须靠 stats[3] 拦截并回退到 + // host/fetch 路径；这里只保证「绝不静默」。 + new_slot[e] = 0; + ++unplaced; + continue; + } + new_slot[e] = slot; + placements.emplace_back(e, slot); + } + // pass3：回填 miss 位置。 + int misspos = 0; + for (size_t i = 0; i < n; ++i) { + if (local[i] < 0) { + auto it = new_slot.find(static_cast(ip[i])); + local[i] = (it != new_slot.end()) ? it->second : 0; + ++misspos; + } + } + stats[0] = hitpos; stats[1] = misspos; stats[2] = static_cast(placements.size()); + stats[3] = unplaced; + return local; +} + +// demand 全接管：inds 惰性(内部 eval 一次=1 次同步)；side_gen 指定侧区代；pool_list 为 _segs 顺序 +// 的 per-key 池数组(已 eval、指针稳定)。返回 local(int32, inds.shape)。 +mx::array demand_dual( + const mx::array& inds, const std::vector& pool_list, + const std::vector& seg_nbytes, int layer, int side_gen, const std::string& path, + int stride, int cap, bool lfu, int decay_interval, mx::StreamOrDevice s = {}) { + if (seg_nbytes.size() != pool_list.size()) + throw std::invalid_argument("demand_dual: seg_nbytes.size() != pool_list.size()"); + double t0 = g_dt_on ? dt_now_us() : 0; + // 关键:inds 常是 argpartition(...)[..., -k:] 切片 → 非连续 strided 视图(每行按父数组 stride E + // 偏移)。若直接按连续读 data(),仅首行(token0)正确,后续 token 读到错位内存 → 装错专家、seq≥2 全错 + // (投机 verify 全体受害)。contiguous() 强制物化为连续,读取才逐位正确。 + mx::array ids = mx::contiguous(inds); + ids.eval(); // 唯一同步 + size_t n = ids.size(); + const uint32_t* ip = ids.data(); + double t1 = g_dt_on ? dt_now_us() : 0; + std::vector ptrs; + ptrs.reserve(pool_list.size()); + for (auto& a0 : pool_list) { mx::array a = a0; a.eval(); ptrs.push_back(a.data()); } + double t2 = g_dt_on ? dt_now_us() : 0; + std::unordered_map side = sideregion_snapshot(layer, side_gen); // 侧区快照(该代) + double ta = g_dt_on ? dt_now_us() : 0; + long stats[4]; + std::vector local; + std::vector> placements; // (expert, slot)：锁外并行落盘 + double tb; + { + std::lock_guard lk(g_real_mutex); + tb = g_dt_on ? dt_now_us() : 0; + RealLayer& c = g_real[layer]; + real_ensure_locked(c, cap); + local = demand_core_locked(c, ip, n, side, lfu, decay_interval, placements, stats); + } + // 锁外：把 miss 专家的字节 pread 落真实区槽。复刻基线并发模型——多 miss 派给 BgReader worker + // 并行 pread（高优队列、直写池段行，无主线程 tmp/memcpy），本层等待其完成（并行 → 远快于串行）。 + static const bool kSkipIO = []() { + const char* e = std::getenv("DEMAND_SKIP_IO"); + return e && e[0] == '1'; + }(); + if (!placements.empty() && !kSkipIO) { + std::vector seg_off(seg_nbytes.size()), seg_nb(seg_nbytes.size()); + long acc = 0; + for (size_t k = 0; k < seg_nbytes.size(); ++k) { + seg_off[k] = acc; seg_nb[k] = seg_nbytes[k]; acc += seg_nbytes[k]; + } + std::vector tickets; + tickets.reserve(placements.size()); + for (auto& pr : placements) { + long tk = g_demand_ticket.fetch_add(1); + bg_pread_into_pool(pool_list, seg_off, seg_nb, pr.second, pr.first, path, + static_cast(stride), tk, /*prio=*/1, + /*nocache=*/false); // demand route 读走 cache：段偏移非页对齐 + tickets.push_back(tk); + } + for (long tk : tickets) bg_reader_wait(tk); // 并行 pread 完成 → 池槽字节就绪 + } + double t3 = g_dt_on ? dt_now_us() : 0; + { + std::lock_guard lk(g_dstat_mutex); + g_d_last[0] = stats[0]; g_d_last[1] = stats[1]; g_d_last[2] = stats[2]; + g_d_last[3] = (stats[1] == 0) ? 0 : 1; + g_d_last[4] = stats[3]; + } + if (stats[3] > 0) { + static std::atomic warned{0}; + long k = warned.fetch_add(1); + if (k < 8 || k % 1000 == 0) + fprintf(stderr, + "[DEMAND_DUAL] 错算：layer=%d 有 %ld 个专家分不到槽(落 0 号槽)。" + "inds=%zu cap=%d pinned=%d —— 该前向输出不可信,请调小 PREFILL_CHUNK " + "或调大 EXPERT_SLOTS / 调小 AUTOPIN_BUDGET_FRAC。\n", + layer, stats[3], n, cap, real_pinned_count(layer)); + } + mx::array out = mx::array(local.data(), ids.shape(), mx::int32); + if (g_dt_on) { + double t4 = dt_now_us(); + g_dt[0] += t1 - t0; g_dt[1] += t2 - t1; g_dt[2] += ta - t2; + g_dt[3] += tb - ta; g_dt[4] += t3 - tb; g_dt[5] += t4 - t3; + } + return out; +} + +// 取本次 demand 统计 [hitpos, misspos, loads, fallback01, unplaced]（主线程串行，安全）。 +std::vector demand_last_stats() { + std::lock_guard lk(g_dstat_mutex); + return {g_d_last[0], g_d_last[1], g_d_last[2], g_d_last[3], g_d_last[4]}; +} + +// 测试壳：纯状态推进(不 pread/不侧区)，把 experts_flat 当一次 demand 的 inds，返回 local 槽位。 +// 供 LFU 驱逐语义逐步等价单测(与 Python 参考对拍)。 +std::vector real_debug_place(int layer, const std::vector& experts_flat, int cap, + bool lfu, int decay_interval) { + std::vector u(experts_flat.begin(), experts_flat.end()); + long stats[4]; + std::vector> placements; + std::lock_guard lk(g_real_mutex); + RealLayer& c = g_real[layer]; + real_ensure_locked(c, cap); + std::unordered_map empty_side; + auto local = demand_core_locked(c, u.data(), u.size(), empty_side, lfu, decay_interval, + placements, stats); + return std::vector(local.begin(), local.end()); +} diff --git a/native/ext/pool/demand.h b/native/ext/pool/demand.h new file mode 100644 index 0000000..9794480 --- /dev/null +++ b/native/ext/pool/demand.h @@ -0,0 +1,29 @@ +// Phase 2 方案B：真实区槽状态 C++ 全接管（1 次同步版）+ demand_dual。 +// 复刻 Python ResidentExpertPool 的 free/LFU 语义，成为 dual-source decode 真实区唯一权威。 +#pragma once +#include "../common.h" + +void real_init(int layer, int cap); // 初始化某层真实区(cap 槽全空闲)，幂等 +std::vector real_region_contents(int layer); // [expert, slot, ...] +int real_region_count(int layer); +void real_reset(); +// AUTOPIN：注册 pinned 热专家(驱逐免选)，返回平行槽位(-1=分不到)；前置 real_init。 +std::vector real_pin(int layer, const std::vector& experts); +// 该层 pinned(永不可驱逐)专家数。调用方据此算「可安放的唯一专家上限」= cap − pinned， +// 用于在超容量前把该前向分流到 host/fetch 路径（超了会落 0 号槽 → 逐位错算）。 +int real_pinned_count(int layer); +// AUTOPIN：导出各层 LFU 累计频次，扁平 [layer, expert, count, ...]。 +std::vector real_freq_dump(); +// demand 全接管：inds 惰性(内部 eval 一次)；side_gen 侧区代；pool_list 为 _segs 顺序池数组。 +mx::array demand_dual( + const mx::array& inds, const std::vector& pool_list, + const std::vector& seg_nbytes, int layer, int side_gen, const std::string& path, + int stride, int cap, bool lfu, int decay_interval, mx::StreamOrDevice s); +// [hitpos, misspos, loads, fallback01, unplaced] +// unplaced：本次分不到槽、被迫落 0 号槽的唯一专家数。>0 即该前向逐位错算(必须为 0)。 +std::vector demand_last_stats(); +std::vector demand_timings(); // [inds_eval, pool_eval, state, build] us +void demand_timing_enable(bool on); +// 测试壳：纯状态推进(不 pread/不侧区)，返回 local 槽位；供 LFU 驱逐等价对拍。 +std::vector real_debug_place(int layer, const std::vector& experts_flat, int cap, + bool lfu, int decay_interval); diff --git a/native/ext/pool/owned_pool.cpp b/native/ext/pool/owned_pool.cpp new file mode 100644 index 0000000..ba0ff1d --- /dev/null +++ b/native/ext/pool/owned_pool.cpp @@ -0,0 +1,89 @@ +// [5] Route 3 Phase 1 底座：C++ 拥有的池 buffer。 +// 用 mx::allocator::malloc 分配 buffer、no-op deleter 建叶子 mx.array（C++ 经 g_owned_bufs +// 持有 Buffer 句柄，进程内不释放）。因 C++ 独占持有、MLX 只读，该 buffer 永不被 MLX +// donation/迁移（spike 已证），故侧区/demand 的后台异步 pread 可安全直写、消费侧读同一块。 +#include "owned_pool.h" + +static std::mutex g_owned_mutex; +static std::vector g_owned_bufs; + +static mx::Dtype dtype_from_str(const std::string& s) { + if (s == "uint32") return mx::uint32; + if (s == "uint16") return mx::uint16; + if (s == "uint8") return mx::uint8; + if (s == "int32") return mx::int32; + if (s == "int16") return mx::int16; + if (s == "bfloat16") return mx::bfloat16; + if (s == "float16") return mx::float16; + if (s == "float32") return mx::float32; + throw std::runtime_error("pool_owned_zeros: unsupported dtype " + s); +} + +mx::array pool_owned_zeros(const std::vector& shape, const std::string& dtype) { + mx::Dtype dt = dtype_from_str(dtype); + mx::Shape shp(shape.begin(), shape.end()); + size_t n = 1; + for (int d : shape) n *= static_cast(d); + size_t nbytes = n * static_cast(mx::size_of(dt)); + auto buf = mx::allocator::malloc(nbytes); + std::memset(buf.raw_ptr(), 0, nbytes); + { + std::lock_guard lk(g_owned_mutex); + g_owned_bufs.push_back(buf); // C++ 持有，保证进程内不被释放 + } + return mx::array(buf, shp, dt, [](mx::allocator::Buffer) {}); +} + +// demand 真实区落池：把一批已加载专家段 memcpy 进 owned 池行（无 MLX scatter → pool buffer 永不重绑）。 +// pool_list[i]：第 i 个 key 的池数组；srcs_flat 长 K*m，key-major：[k0行0,k0行1,...,k1行0,...]； +// slots[j]：第 j 个专家的目标物理行。CPU 端直写，调用点须保证此刻池未被 GPU 并发读该行。 +void pool_write_rows(const std::vector& pool_list, + const std::vector& srcs_flat, + const std::vector& slots) { + int K = static_cast(pool_list.size()); + int m = static_cast(slots.size()); + if (m == 0) return; + if (static_cast(srcs_flat.size()) != K * m) + throw std::runtime_error("pool_write_rows: srcs_flat 长度须 == K*m"); + for (int i = 0; i < K; ++i) { + mx::array p = pool_list[i]; + p.eval(); + uint8_t* base = p.data(); + for (int j = 0; j < m; ++j) { + mx::array s = srcs_flat[static_cast(i) * m + j]; + s.eval(); + size_t nb = s.nbytes(); + std::memcpy(base + static_cast(slots[j]) * nb, s.data(), nb); + } + } +} + +// 同上，但源是每 key 预堆叠的 (m,*shape) 整块 stacked_list[i]，按行 memcpy 到 slots[j]。 +void pool_write_stacked(const std::vector& pool_list, + const std::vector& stacked_list, + const std::vector& slots) { + int K = static_cast(pool_list.size()); + int m = static_cast(slots.size()); + if (m == 0) return; + for (int i = 0; i < K; ++i) { + mx::array p = pool_list[i]; + p.eval(); + mx::array st = stacked_list[i]; + st.eval(); + uint8_t* base = p.data(); + const uint8_t* src = st.data(); + size_t rownb = st.nbytes() / static_cast(m); // stacked 为 (m,*shape) → 每行字节 + for (int j = 0; j < m; ++j) + std::memcpy(base + static_cast(slots[j]) * rownb, + src + static_cast(j) * rownb, rownb); + } +} + +// 临时诊断：返回某 mx.array 底层 buffer 的原始数据指针（uintptr）。用于对拍 +// 「C++ 侧区 memcpy 写入的 buffer 指针」与「Python consume/verify 读到的 pool buffer 指针」 +// 是否同一块——若不同即证明 MLX 在两者之间重分配了池 buffer，raw 写入被落单。 +uintptr_t array_data_ptr(const mx::array& a) { + mx::array b = a; + b.eval(); + return reinterpret_cast(b.data()); +} diff --git a/native/ext/pool/owned_pool.h b/native/ext/pool/owned_pool.h new file mode 100644 index 0000000..d1cb562 --- /dev/null +++ b/native/ext/pool/owned_pool.h @@ -0,0 +1,14 @@ +// Route 3 Phase 1 底座：C++ 拥有的池 buffer + 直写（替代 mx.zeros 建池 + 消费侧 MLX scatter）。 +#pragma once +#include "../common.h" + +// C++ 拥有的池 buffer 包成 mx.array（no-op deleter，进程内持有、永不迁移）。 +// 供侧区/demand 后台 pread 安全直写；替代原 mx.zeros 建池 + 消费侧 MLX scatter。 +mx::array pool_owned_zeros(const std::vector& shape, const std::string& dtype); +// demand 真实区落池：把已加载专家段直接 memcpy 进 owned 池行（无 MLX scatter，保 buffer 稳定）。 +void pool_write_rows(const std::vector& pool_list, + const std::vector& srcs_flat, const std::vector& slots); +void pool_write_stacked(const std::vector& pool_list, + const std::vector& stacked_list, const std::vector& slots); +// 诊断：mx.array 底层 buffer 原始指针（uintptr），用于对拍侧区写入 buffer 与 consume 读到 buffer 是否同一块。 +uintptr_t array_data_ptr(const mx::array& a); diff --git a/native/ext/pool/side_region.cpp b/native/ext/pool/side_region.cpp new file mode 100644 index 0000000..a49337d --- /dev/null +++ b/native/ext/pool/side_region.cpp @@ -0,0 +1,389 @@ +// [4] 段散写持久侧区缓存（zero-copy dual-source 默认路径）。 +// 与持久 staging 缓存同构，但目标是“多个结构化 per-key 池数组”：每个池数组形如 +// (cap+spec, ...)，行 [base_row, base_row+spec) 为侧区。命中缺口时 pread blob 整行， +// 再把该行内按固定顺序拼接的各段 memcpy 进对应 per-key 数组的同一物理行。 +#include "side_region.h" +#include "../io/blob_io.h" +#include "../io/bg_reader.h" + +#include +#include +#include +#include +#include +#include + +struct SideLayer { + std::map e2r; // expert -> 物理侧区行 [base_row, base_row+spec) + std::vector free_rows; + std::map freq; // expert -> 预测频次(LFU 分数;仅 SIDEREGION_LFU 用) + bool inited = false; + int base = 0; // 侧区起始物理行 base_row + int spec = 0; // 侧区行数 spec_slots +}; +static std::mutex g_side_mutex; +static std::map, SideLayer> g_side; // 键 (layer, gen)：双缓冲两代独立 + +// 侧区 fill 在途计数：eval_gpu 提交预取时 +1，后台 read_publish 写完字节后 -1。 +// 消费方在前向开头 sideregion_drain() 排空上一前向的 fill，保证被消费的侧区行字节 +// 已完全写好（含 GPU 完成回调滞后的情形）→ 消灭「GPU 消费 kernel 读到半写侧区行」竞态。 +static std::atomic g_side_inflight{0}; +static std::mutex g_side_drain_mutex; +static std::condition_variable g_side_drain_cv; +static inline void side_inflight_done() { + { + std::lock_guard lk(g_side_drain_mutex); + g_side_inflight.fetch_sub(1); + } + g_side_drain_cv.notify_all(); +} +void sideregion_drain() { + std::unique_lock lk(g_side_drain_mutex); + g_side_drain_cv.wait(lk, [] { return g_side_inflight.load() == 0; }); +} + +// 临时诊断：只追踪指定 (layer,row) 的所有账本/字节事件（SIDE_TRACE_LAYER/ROW）。 +static inline bool side_trace_hit(int layer, int row) { + const char* le = std::getenv("SIDE_TRACE_LAYER"); + const char* re = std::getenv("SIDE_TRACE_ROW"); + if (!le || !re) return false; + return layer == atoi(le) && row == atoi(re); +} + +// 临时诊断：跨线程全局事件序号 + 线程短 id，用于把 reserve(Metal 回调线程)/read_publish(bg 线程)/ +// sideregion_kv(主线程) 的事件按发生顺序排出来（root-cause 取证，SIDE_TRACE_* 命中时才用）。 +static std::atomic g_side_ev{0}; +static inline unsigned side_tid() { + return static_cast( + std::hash{}(std::this_thread::get_id()) & 0xffff); +} + +class PrefetchPoolSideRegionPrimitive : public mx::Primitive { + public: + PrefetchPoolSideRegionPrimitive(mx::Stream s, std::vector seg_nbytes, int layer, int gen, + std::string path, size_t stride, std::vector resident, + int spec_slots, int base_row) + : Primitive(s), seg_(std::move(seg_nbytes)), layer_(layer), gen_(gen), path_(std::move(path)), + stride_(stride), resident_(std::move(resident)), spec_(spec_slots), base_(base_row) {} + const char* name() const override { return "PrefetchPoolSideRegionPrimitive"; } + + void eval_cpu(const std::vector& in, std::vector& out) override { + out[0].set_data(mx::allocator::malloc(out[0].nbytes())); + run(in); + } + void eval_gpu(const std::vector& in, std::vector& out) override { + out[0].set_data(mx::allocator::malloc(out[0].nbytes())); + std::vector ptrs; + ptrs.reserve(in.size() - 1); + for (size_t i = 1; i < in.size(); ++i) { + mx::array a = in[i]; // 非 const 拷贝才能取可写指针 + ptrs.push_back(a.data()); + } + mx::array ids = in[0]; + const uint32_t* idp = ids.data(); + size_t n = ids.size(); + std::vector seg = seg_; + int layer = layer_; + int gen = gen_; + std::string path = path_; + size_t stride = stride_; + std::vector resident = resident_; + int spec = spec_, base = base_; + auto& enc = mx::metal::get_command_encoder(stream()); + MTL::CommandBuffer* cb = enc.get_command_buffer(); + // 提交即计在途：在 eval_gpu（预取提交）时 +1，直到后台字节写完才 -1。这样消费方前向开头 + // sideregion_drain() 能等到「即使 GPU 完成回调尚未触发」的 fill，闭合跨前向的写-读竞态。 + g_side_inflight.fetch_add(1); + // in 按值捕获 → 保活 expert_ids 与所有池数组 buffer；idp/ptrs 在回调里指针有效。 + cb->addCompletedHandler( + [in, ptrs, seg, idp, n, layer, gen, path, stride, resident, spec, base](MTL::CommandBuffer*) { + // 阶段1（回调线程、持锁极短）：读惰性 id（此刻已算完）、预留侧区行。 + // 必须在回调里——id 只有 command buffer 完成后才有效。 + auto to_read = reserve(idp, n, layer, gen, resident, spec, base); + if (to_read.empty()) { + side_inflight_done(); // 无缺口可读：立即消账，避免 drain 空等 + return; + } + // 诊断门控 SIDEREGION_SYNC=1：回调内同步 pread+memcpy+publish（不派 bg）， + // 消除「异步 bg fill 与下一前向 gather 竞态」这一变量，用于 systematic-debugging 取证。 + static const bool kSync = []() { + const char* e = std::getenv("SIDEREGION_SYNC"); + return e && e[0] == '1'; + }(); + if (kSync) { + read_publish(ptrs, seg, to_read, path, stride, layer, gen); + side_inflight_done(); + return; + } + // 阶段2+3 派给自由后台线程：~40MB pread + memcpy + 发布 e2r 脱离 Metal 回调线程， + // 与主 stream 后续层计算、多层预取互相并发 → 真正恢复 I/O/计算重叠。 + // 闭包再次按值捕获 in：保活池/ids buffer 到后台读完（ptrs 指向其内存）。 + bg_submit_task([in, ptrs, seg, to_read, path, stride, layer, gen]() { + read_publish(ptrs, seg, to_read, path, stride, layer, gen); + side_inflight_done(); // 字节写完才消账 → drain 保证被消费行已就绪 + }); + }); + } + + private: + void run(const std::vector& in) { + std::vector ptrs; + ptrs.reserve(in.size() - 1); + for (size_t i = 1; i < in.size(); ++i) { + mx::array a = in[i]; + a.eval(); // 与 PrefetchStagingCachedPrimitive::eval_cpu 一致：先物化输入 + ptrs.push_back(a.data()); + } + mx::array ids = in[0]; + ids.eval(); + // CPU 路径同步执行（测试/无 GPU 时）：预留 + 读发布一气呵成。 + auto to_read = reserve(ids.data(), ids.size(), layer_, gen_, resident_, spec_, base_); + read_publish(ptrs, seg_, to_read, path_, stride_, layer_, gen_); + } + + // 阶段1：过滤常驻/去重 → 淘汰 ∉P 的旧行 → 为缺口预留物理行（出 free，暂不入 e2r， + // 避免消费者在字节写好前看到 e2r 命中而 gather 到脏字节）。返回 (expert, 预留行)。 + static std::vector> reserve( + const uint32_t* idp, size_t n, int layer, int gen, const std::vector& resident, + int spec, int base) { + const char* lfu_env = std::getenv("SIDEREGION_LFU"); // 每次读,便于测试切换 + // 默认开:持久 LFU 单缓冲=生产路径(cli 默认)。仅显式 SIDEREGION_LFU=0 回退旧 legacy 双缓冲。 + bool lfu = !lfu_env || lfu_env[0] != '0'; + std::unordered_set res(resident.begin(), resident.end()); + std::vector P; + std::unordered_set Pset, seen; + for (size_t i = 0; i < n; ++i) { + int e = static_cast(idp[i]); + if (res.count(e) || !seen.insert(e).second) continue; + P.push_back(e); + Pset.insert(e); + } + std::vector> to_read; + std::lock_guard lk(g_side_mutex); + SideLayer& c = g_side[{layer, gen}]; + if (!c.inited) { + for (int r = 0; r < spec; ++r) c.free_rows.push_back(base + r); + c.base = base; + c.spec = spec; + c.inited = true; + } + if (!lfu) { + // 旧行为:∉P 全弃(一次性预取批)。 + for (auto it = c.e2r.begin(); it != c.e2r.end();) { + if (!Pset.count(it->first)) { + c.free_rows.push_back(it->second); + it = c.e2r.erase(it); + } else { + ++it; + } + } + for (int e : P) { + if (c.e2r.count(e) || c.free_rows.empty()) continue; + to_read.emplace_back(e, c.free_rows.back()); + c.free_rows.pop_back(); + } + return to_read; + } + // LFU 持久:∉P 不清;再预测命中已驻专家 freq+1(越常预测越热)。 + for (int e : P) { + if (c.e2r.count(e)) c.freq[e] += 1; + } + for (int e : P) { + if (c.e2r.count(e)) continue; // 已驻,跳过(不重读) + int row; + if (!c.free_rows.empty()) { + row = c.free_rows.back(); + c.free_rows.pop_back(); + if (side_trace_hit(layer, row)) + fprintf(stderr, "[SIDE_TRACE ev=%llu tid=%u] L%d gen%d RESERVE_FROM_FREE row=%d expert=%d\n", + (unsigned long long)g_side_ev.fetch_add(1), side_tid(), layer, gen, row, e); + } else { + // free 空:LFU 淘汰 e2r 中 freq 最小且 ∉P 者(tie-break:最小 expert id)。 + int victim = -1; + uint32_t best = 0; + for (auto& kv : c.e2r) { + if (Pset.count(kv.first)) continue; // 不淘本步要用的 + uint32_t f = c.freq.count(kv.first) ? c.freq[kv.first] : 0; + if (victim < 0 || f < best || (f == best && kv.first < victim)) { + victim = kv.first; + best = f; + } + } + if (victim < 0) continue; // 全是 P 热,无可淘 → 本步不读 + row = c.e2r[victim]; + c.e2r.erase(victim); + c.freq.erase(victim); + if (side_trace_hit(layer, row)) + fprintf(stderr, "[SIDE_TRACE ev=%llu tid=%u] L%d gen%d EVICT_REUSE row=%d victim=%d newExpert=%d\n", + (unsigned long long)g_side_ev.fetch_add(1), side_tid(), layer, gen, row, victim, e); + } + to_read.emplace_back(e, row); + } + // ===== 临时诊断（SIDE_AUDIT=1）：reserve 结束时审计侧区行账本自洽性 ===== + if (std::getenv("SIDE_AUDIT")) side_audit(c, layer, gen, "reserve", to_read); + return to_read; + } + + // 审计：检测「一行被两专家占用」「free 与 e2r 行重叠」「to_read 行仍被 e2r 占用」。 + static void side_audit(SideLayer& c, int layer, int gen, const char* where, + const std::vector>& to_read) { + std::map row_owner; // row -> expert + for (auto& p : c.e2r) { + auto it = row_owner.find(p.second); + if (it != row_owner.end()) + fprintf(stderr, "[SIDE_AUDIT] %s L%d gen%d DOUBLE_OWN row=%d experts=%d,%d\n", + where, layer, gen, p.second, it->second, p.first); + row_owner[p.second] = p.first; + } + std::unordered_set free_set(c.free_rows.begin(), c.free_rows.end()); + for (auto& p : c.e2r) + if (free_set.count(p.second)) + fprintf(stderr, "[SIDE_AUDIT] %s L%d gen%d FREE_E2R_OVERLAP row=%d owned_by=%d\n", + where, layer, gen, p.second, p.first); + if (c.free_rows.size() != free_set.size()) + fprintf(stderr, "[SIDE_AUDIT] %s L%d gen%d FREE_DUP free_n=%zu uniq=%zu\n", + where, layer, gen, c.free_rows.size(), free_set.size()); + for (auto& pr : to_read) + if (row_owner.count(pr.second)) + fprintf(stderr, "[SIDE_AUDIT] %s L%d gen%d TOREAD_LIVE_ROW row=%d assigned_to=%d still_owned_by=%d\n", + where, layer, gen, pr.second, pr.first, row_owner[pr.second]); + } + + // 阶段2+3：pread blob 整行 + 各段 memcpy 直写进对应 per-key 池数组的物理侧区行（不持锁）， + // 写完后持锁发布 e2r。ptrs[i] 为第 i 个池 key 数组的 buffer 指针（C++ 拥有、地址恒定不迁移， + // 由 Route 3 底座保证——见 pool_owned_zeros），故后台异步直写安全、消费侧读同一 buffer。 + // 在后台线程跑：与计算并发，且不阻塞消费侧的 sideregion_kv。 + static void read_publish(const std::vector& ptrs, const std::vector& seg, + const std::vector>& to_read, + const std::string& path, size_t stride, int layer, int gen) { + // 段在 blob 记录内的偏移（段顺序 = seg 顺序 = ptrs/池 key 顺序）。 + std::vector seg_off(seg.size(), 0); + size_t acc = 0; + for (size_t i = 0; i < seg.size(); ++i) { seg_off[i] = acc; acc += static_cast(seg[i]); } + int fd = open_blob_nocache(path.c_str()); + if (fd < 0) { // 失败则把预留行还回 free + std::lock_guard lk(g_side_mutex); + SideLayer& c = g_side[{layer, gen}]; + for (auto& pr : to_read) c.free_rows.push_back(pr.second); + return; + } + std::vector rec(stride); // 整条 blob 记录临时缓冲 + std::vector> done; + for (auto& pr : to_read) { + int e = pr.first, row = pr.second; + if (::pread(fd, rec.data(), stride, static_cast(static_cast(e) * stride)) != + static_cast(stride)) { + std::lock_guard lk(g_side_mutex); // 读失败：行还回 free + g_side[{layer, gen}].free_rows.push_back(row); + continue; + } + // 各段 memcpy 进对应 per-key 池数组的物理行 row（(n_slots,*shape) 布局 → 行偏移 = row*seg[i]）。 + for (size_t i = 0; i < seg.size(); ++i) + std::memcpy(ptrs[i] + static_cast(row) * static_cast(seg[i]), + rec.data() + seg_off[i], static_cast(seg[i])); + if (side_trace_hit(layer, row)) + fprintf(stderr, "[SIDE_TRACE ev=%llu tid=%u] L%d gen%d WRITEPOOL row=%d expert=%d\n", + (unsigned long long)g_side_ev.fetch_add(1), side_tid(), layer, gen, row, e); + done.emplace_back(e, row); + } + ::close(fd); + { + std::lock_guard lk(g_side_mutex); + SideLayer& c = g_side[{layer, gen}]; + for (auto& pr : done) { // 字节就绪后才发布 e2r + c.e2r[pr.first] = pr.second; + if (!c.freq.count(pr.first)) c.freq[pr.first] = 1; // 新专家初始 freq + if (side_trace_hit(layer, pr.second)) + fprintf(stderr, "[SIDE_TRACE ev=%llu tid=%u] L%d gen%d PUBLISH row=%d expert=%d\n", + (unsigned long long)g_side_ev.fetch_add(1), side_tid(), layer, gen, pr.second, + pr.first); + } + if (std::getenv("SIDE_AUDIT")) side_audit(c, layer, gen, "publish", {}); + } + } + std::vector seg_; + int layer_; + int gen_; + std::string path_; + size_t stride_; + std::vector resident_; + int spec_; + int base_; +}; + +mx::array prefetch_pool_sideregion( + const std::vector& pool_list, const std::vector& seg_nbytes, + const mx::array& expert_ids, int layer, const std::string& path, int stride, + const std::vector& resident, int spec_slots, int base_row, int gen, + mx::StreamOrDevice s = {}) { + // 防御：段数必须与池数组个数一一对应（每段映射唯一一个 per-key 数组）。 + if (seg_nbytes.size() != pool_list.size()) { + throw std::invalid_argument( + "prefetch_pool_sideregion: seg_nbytes.size()=" + std::to_string(seg_nbytes.size()) + + " != pool_list.size()=" + std::to_string(pool_list.size())); + } + // 防御：各段字节之和必须等于 stride（blob 记录恰好是各段的拼接）。 + size_t seg_sum = 0; + for (int b : seg_nbytes) seg_sum += static_cast(b); + if (seg_sum != static_cast(stride)) { + throw std::invalid_argument( + "prefetch_pool_sideregion: sum(seg_nbytes)=" + std::to_string(seg_sum) + + " != stride=" + std::to_string(stride)); + } + std::vector inputs; + inputs.push_back(expert_ids); + for (auto& a : pool_list) inputs.push_back(a); + return mx::array( + mx::Shape{1}, mx::uint8, + std::make_shared( + mx::to_stream(s), seg_nbytes, layer, gen, path, static_cast(stride), resident, + spec_slots, base_row), + inputs); +} + +std::vector sideregion_contents(int layer, int gen) { + std::lock_guard lk(g_side_mutex); + std::vector out; + auto it = g_side.find({layer, gen}); + if (it != g_side.end()) + for (auto& p : it->second.e2r) { out.push_back(p.first); out.push_back(p.second); } + return out; +} + +// 侧区 e2r → 两个 device mx.array (keys uint32, vals int32),直接在 C++ 从 map 建连续 buffer。 +// 消掉 Python 侧 dict 构建 + list(...)→mx.array 的每层 host 胶水。 +std::pair sideregion_kv(int layer, int gen) { + std::vector keys; + std::vector vals; + { + std::lock_guard lk(g_side_mutex); + auto it = g_side.find({layer, gen}); + if (it != g_side.end()) { + keys.reserve(it->second.e2r.size()); + vals.reserve(it->second.e2r.size()); + for (auto& p : it->second.e2r) { + keys.push_back(static_cast(p.first)); + vals.push_back(static_cast(p.second)); + if (side_trace_hit(layer, p.second)) + fprintf(stderr, "[SIDE_TRACE ev=%llu tid=%u] L%d gen%d CONSUME_KV row=%d expert=%d\n", + (unsigned long long)g_side_ev.fetch_add(1), side_tid(), layer, gen, p.second, p.first); + } + } + } + int n = static_cast(keys.size()); + return {mx::array(keys.data(), mx::Shape{n}, mx::uint32), + mx::array(vals.data(), mx::Shape{n}, mx::int32)}; +} + +void sideregion_reset() { + std::lock_guard lk(g_side_mutex); + g_side.clear(); +} + +std::unordered_map sideregion_snapshot(int layer, int gen) { + std::unordered_map side; + std::lock_guard lk(g_side_mutex); + auto it = g_side.find({layer, gen}); + if (it != g_side.end()) for (auto& p : it->second.e2r) side[p.first] = p.second; + return side; +} diff --git a/native/ext/pool/side_region.h b/native/ext/pool/side_region.h new file mode 100644 index 0000000..209935b --- /dev/null +++ b/native/ext/pool/side_region.h @@ -0,0 +1,21 @@ +// 段散写持久侧区缓存（zero-copy dual-source 默认路径）：keep/evict/read-delta， +// 命中缺口时 pread blob 行并把各段 memcpy 进对应 per-key 池数组的 (base_row+cache_row) 行。 +#pragma once +#include "../common.h" +#include + +mx::array prefetch_pool_sideregion( + const std::vector& pool_list, const std::vector& seg_nbytes, + const mx::array& expert_ids, int layer, const std::string& path, int stride, + const std::vector& resident, int spec_slots, int base_row, int gen, + mx::StreamOrDevice s); +std::vector sideregion_contents(int layer, int gen); // [expert, phys_row, ...] +std::pair sideregion_kv(int layer, int gen); // (keys uint32, vals int32) device 数组 +void sideregion_reset(); +// 排空侧区在途 fill：阻塞直到所有已提交的侧区预取字节写完。消费方在前向开头调用， +// 保证被消费的侧区行字节已完全写好（闭合异步写-读竞态）。 +void sideregion_drain(); + +// 内部接口（不绑 Python）：取某层某代侧区 e2r 快照，供 demand_dual 在自己的核心状态机里 +// 叠加侧区命中。等价于在 g_side 锁下拷贝 e2r（与原 demand_dual 内联循环一字不差）。 +std::unordered_map sideregion_snapshot(int layer, int gen); diff --git a/native/ext/prefetch/prefetch.cpp b/native/ext/prefetch/prefetch.cpp new file mode 100644 index 0000000..883addf --- /dev/null +++ b/native/ext/prefetch/prefetch.cpp @@ -0,0 +1,217 @@ +// [2] 轻量预取(无 staging，仅预热 page cache) + [3] staging 版 miss→hit + STAGING_HPROF 探针。 +#include "prefetch.h" +#include "../io/bg_reader.h" + +#include +#include +#include +#include +#include +#include +#include + +// ====== [2] 轻量预取(无 staging 模式)：GPU 完成回调里读 inds、pread 预热 page cache ====== +// 命门验证:回调在 command buffer 完成后触发,此时 inds 已算完,读到的是正确值。 +class PrefetchOnCompletePrimitive : public mx::Primitive { + public: + PrefetchOnCompletePrimitive(mx::Stream s, std::string path, size_t stride, bool do_read) + : Primitive(s), path_(std::move(path)), stride_(stride), do_read_(do_read) {} + const char* name() const override { return "PrefetchOnCompletePrimitive"; } + + void eval_cpu(const std::vector& inputs, std::vector& outputs) override { + outputs[0].set_data(mx::allocator::malloc(outputs[0].nbytes())); + mx::array ids = inputs[0]; + ids.eval(); + record_and_read(ids.data(), ids.size(), path_, stride_, do_read_); + } + + void eval_gpu(const std::vector& inputs, std::vector& outputs) override { + outputs[0].set_data(mx::allocator::malloc(outputs[0].nbytes())); + mx::array ids = inputs[0]; // by-value：保活到回调结束 + const uint32_t* ptr = ids.data(); // 指针此刻有效（buffer 已分配），值待 GPU 算完 + size_t n = ids.size(); + std::string path = path_; + size_t stride = stride_; + bool do_read = do_read_; + auto& enc = mx::metal::get_command_encoder(stream()); + MTL::CommandBuffer* cb = enc.get_command_buffer(); + cb->addCompletedHandler([ids, ptr, n, path, stride, do_read](MTL::CommandBuffer*) { + // 此时 buffer 已完成 → ptr 指向已算好的 inds 值。ids 按值捕获保证 buffer 不被释放。 + record_and_read(ptr, n, path, stride, do_read); + }); + } + + private: + static void record_and_read(const uint32_t* p, size_t n, const std::string& path, + size_t stride, bool do_read) { + int fd = (do_read && !path.empty()) ? ::open(path.c_str(), O_RDONLY) : -1; + static thread_local std::vector buf; + if (do_read && buf.size() < stride) buf.resize(stride); + for (size_t i = 0; i < n; ++i) { + int e = static_cast(p[i]); + if (fd >= 0) ::pread(fd, buf.data(), stride, static_cast(static_cast(e) * stride)); + } + if (fd >= 0) ::close(fd); + } + std::string path_; + size_t stride_; + bool do_read_; +}; + +// 返回一个 dummy 输出（挂进图里，eval 时触发上面的回调）。 +mx::array prefetch_on_complete( + const mx::array& expert_ids, + const std::string& path, + int stride, + bool do_read = true, + mx::StreamOrDevice s = {}) { + return mx::array( + mx::Shape{1}, + mx::uint8, + std::make_shared( + mx::to_stream(s), path, static_cast(stride), do_read), + std::vector{expert_ids}); +} + +// ---- handler 触发时刻探针(STAGING_HPROF)：记录每次 staging 完成回调被 Metal 触发的时刻 ---- +// 用于实测"完成回调到底扎堆在 eval 尾、还是逐层铺开"。仅诊断用，默认关。 +static std::mutex g_hprof_mutex; +static std::vector> g_hprof_log; // (gen, layer, t_fire_seconds) +static bool g_hprof_on = false; + +static inline double hprof_steady_now() { + return std::chrono::duration( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); +} + +// ====== [3] staging 版 miss→hit：handler pread 进 per-layer staging + 记录 (expert→row) ====== +static std::mutex g_stg_mutex; +// layer -> (gen, [(expert, row)])；handler 原子写 gen+映射，主线程按 gen 匹配 buffer 后 take。 +static std::map>>> g_stg_ready; + +class PrefetchStagingPrimitive : public mx::Primitive { + public: + // 方案B：expert_ids 是"预测宽集合"(top-N，按门控分降序)；resident 是目标层当前常驻专家。 + // handler 在回调里过滤掉常驻、按降序取前 cap 个缺口 pread 进 staging（cap=buffer 行数）。 + // 这样预测可以很宽(高 recall)，而 staging 内存只按 cap 预留(小，覆盖缺口分布即可)。 + PrefetchStagingPrimitive(mx::Stream s, int layer, long gen, std::string path, size_t stride, + std::vector resident, int cap, bool parallel) + : Primitive(s), layer_(layer), gen_(gen), path_(std::move(path)), stride_(stride), + resident_(std::move(resident)), cap_(cap), parallel_(parallel) {} + const char* name() const override { return "PrefetchStagingPrimitive"; } + + void eval_cpu(const std::vector& inputs, std::vector& outputs) override { + outputs[0].set_data(mx::allocator::malloc(outputs[0].nbytes())); + mx::array ids = inputs[0]; ids.eval(); + mx::array stg = inputs[1]; stg.eval(); + fill(ids.data(), ids.size(), stg.data(), layer_, gen_, path_, stride_, + resident_, cap_); + } + void eval_gpu(const std::vector& inputs, std::vector& outputs) override { + outputs[0].set_data(mx::allocator::malloc(outputs[0].nbytes())); + mx::array ids = inputs[0]; + mx::array stg = inputs[1]; + const uint32_t* idp = ids.data(); + uint8_t* sp = stg.data(); + size_t n = ids.size(); + int layer = layer_; long gen = gen_; std::string path = path_; size_t stride = stride_; + std::vector resident = resident_; int cap = cap_; bool parallel = parallel_; + auto& enc = mx::metal::get_command_encoder(stream()); + MTL::CommandBuffer* cb = enc.get_command_buffer(); + cb->addCompletedHandler( + [ids, stg, idp, sp, n, layer, gen, path, stride, resident, cap, parallel](MTL::CommandBuffer*) { + // 探针：记录本回调被 Metal 触发的时刻(= 该层 pread 能开始跑的时刻)。 + if (g_hprof_on) { + double t = hprof_steady_now(); + std::lock_guard lk(g_hprof_mutex); + g_hprof_log.emplace_back(gen, layer, t); + } + // ids/stg 按值捕获保活 buffer；派后台线程时再拷一份保活到 fill 跑完。 + if (parallel) { + bg_submit_task([ids, stg, idp, sp, n, layer, gen, path, stride, resident, cap]() { + fill(idp, n, sp, layer, gen, path, stride, resident, cap); + }); + } else { + fill(idp, n, sp, layer, gen, path, stride, resident, cap); + } + }); + } + + private: + static void fill(const uint32_t* idp, size_t n, uint8_t* stg, + int layer, long gen, const std::string& path, size_t stride, + const std::vector& resident, int cap) { + int fd = ::open(path.c_str(), O_RDONLY); + if (fd < 0) return; + std::unordered_set res(resident.begin(), resident.end()); + std::unordered_set done; // 去重：同一缺口只 pread 一次 + std::vector> ready; + ready.reserve(static_cast(cap)); + int row = 0; // 写入的 staging 行（≤ cap） + for (size_t i = 0; i < n && row < cap; ++i) { + int e = static_cast(idp[i]); + if (res.count(e) || !done.insert(e).second) continue; // 已常驻/已取过 → 跳过 + if (::pread(fd, stg + static_cast(row) * stride, stride, + static_cast(static_cast(e) * stride)) + == static_cast(stride)) { + ready.emplace_back(e, row); + ++row; + } + } + ::close(fd); + std::lock_guard lk(g_stg_mutex); + g_stg_ready[layer] = {gen, std::move(ready)}; // 原子：gen 与映射一起写 + } + int layer_; + long gen_; + std::string path_; + size_t stride_; + std::vector resident_; // 目标层提交时刻的常驻专家快照（过滤用） + int cap_; // staging buffer 行数上限 + bool parallel_; // true: fill 派后台线程池并行；false: 回调线程同步(旧行为) +}; + +mx::array prefetch_into_staging( + const mx::array& staging, const mx::array& expert_ids, int layer, long gen, + const std::string& path, int stride, const std::vector& resident, int cap, + bool parallel, mx::StreamOrDevice s = {}) { + return mx::array( + mx::Shape{1}, mx::uint8, + std::make_shared( + mx::to_stream(s), layer, gen, path, static_cast(stride), resident, cap, parallel), + std::vector{expert_ids, staging}); +} + +// 取走某层就绪记录：[gen, e0,r0,e1,r1,...]（首元素是 generation）；空表示无就绪。 +std::vector prefetch_staging_take(int layer) { + std::lock_guard lk(g_stg_mutex); + std::vector out; + auto it = g_stg_ready.find(layer); + if (it != g_stg_ready.end()) { + out.push_back(it->second.first); // gen + for (auto& p : it->second.second) { out.push_back(p.first); out.push_back(p.second); } + g_stg_ready.erase(it); + } + return out; +} + +// ---- handler 触发时刻探针接口 ---- +void staging_hprof_enable(bool on) { + std::lock_guard lk(g_hprof_mutex); + g_hprof_on = on; + g_hprof_log.clear(); // 开启即清零，便于每次采集干净 +} +double staging_hprof_now() { return hprof_steady_now(); } // 与日志同一时钟，供 Python 标 eval 边界 +// 扁平返回 [gen0,layer0,t0, gen1,layer1,t1, ...]（避开 nanobind tuple caster）。 +std::vector staging_hprof_get() { + std::lock_guard lk(g_hprof_mutex); + std::vector out; + out.reserve(g_hprof_log.size() * 3); + for (auto& r : g_hprof_log) { + out.push_back(static_cast(std::get<0>(r))); + out.push_back(static_cast(std::get<1>(r))); + out.push_back(std::get<2>(r)); + } + return out; +} diff --git a/native/ext/prefetch/prefetch.h b/native/ext/prefetch/prefetch.h new file mode 100644 index 0000000..2d9cc2a --- /dev/null +++ b/native/ext/prefetch/prefetch.h @@ -0,0 +1,23 @@ +// GPU 完成回调预取：算完 inds 后在回调里读 id、pread 预热 page cache 或写 per-layer staging。 +#pragma once +#include "../common.h" + +// 挂 GPU 完成回调：算完 inds 后在回调里读 id 预热 page cache，返回 dummy。 +// （无 staging 的轻量预取模式：仅把预测专家字节读进 page cache，不落池。） +mx::array prefetch_on_complete( + const mx::array& expert_ids, const std::string& path, int stride, + bool do_read, mx::StreamOrDevice s); + +// miss→hit（方案B）：expert_ids 为预测宽集合(降序)，回调按 resident 过滤、取前 cap 个缺口 +// pread 进 per-layer staging buffer(cap=buffer 行数)，并原子记录 (gen,[expert,row])。 +mx::array prefetch_into_staging( + const mx::array& staging, const mx::array& expert_ids, int layer, long gen, + const std::string& path, int stride, const std::vector& resident, int cap, + bool parallel, mx::StreamOrDevice s); +// 取走某层就绪记录：[gen, e0,r0,e1,r1,...]；空表示无就绪。 +std::vector prefetch_staging_take(int layer); + +// handler 触发时刻探针(诊断用)：enable 开关并清零；now 取同时钟当前秒；get 取 (gen,layer,t) 日志。 +void staging_hprof_enable(bool on); +double staging_hprof_now(); +std::vector staging_hprof_get(); // 扁平 [gen,layer,t, ...] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..5a7aa08 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,44 @@ +[project] +name = "sparkle-engine" +version = "0.1.0" +description = "Sparkle：Apple Silicon 上流式 MoE 推理（Qwen3-Next-80B 8bit）" +readme = "README.md" +# 仓库自带 native_moe_ext*.cpython-314*.so，须与扩展 ABI 一致 +requires-python = "==3.14.*" +dependencies = [ + "fastapi>=0.115", + "mlx>=0.31", + "mlx-lm>=0.31", + "numpy>=2.0", + "textual>=0.80", + "uvicorn>=0.30", +] + +[project.scripts] +sparkle = "mlx_streaming.cli:main" + +# 仅重编 native / 跑单测时需要；日常推理用已有 .venv + .so 即可 +[dependency-groups] +dev = [ + "httpx>=0.27", + "nanobind>=2.12.0", + "pytest>=8", + "pytest-asyncio>=0.23", +] + +[[tool.uv.index]] +# 清华 PyPI 镜像 +url = "https://pypi.tuna.tsinghua.edu.cn/simple" +default = true + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["mlx_streaming"] + +[tool.pytest.ini_options] +addopts = "-q" +testpaths = ["mlx_streaming/tests"] +asyncio_mode = "auto" diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..5ba4be0 --- /dev/null +++ b/uv.lock @@ -0,0 +1,1129 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "annotated-doc" +version = "0.0.4" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/57/ba/046ceea27344560984e26a590f90bc7f4a75b06701f653222458922b558c/annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4", size = 7288, upload-time = "2025-11-10T22:07:42.062Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", size = 5303, upload-time = "2025-11-10T22:07:40.673Z" }, +] + +[[package]] +name = "annotated-types" +version = "0.8.0" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, +] + +[[package]] +name = "anyio" +version = "4.13.0" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/19/14/2c5dd9f512b66549ae92767a9c7b330ae88e1932ca57876909410251fe13/anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc", size = 231622, upload-time = "2026-03-24T12:59:09.671Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" }, +] + +[[package]] +name = "certifi" +version = "2026.5.20" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f3/ce/ee2ecad540810a79593028e88299baeae54d346cc7a0d94b6199988b89b1/certifi-2026.5.20.tar.gz", hash = "sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d", size = 135422, upload-time = "2026-05-20T11:46:50.073Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/59/8c/57e832b7af6d7c5abe66eb3fbe3a3a32f4d11ea23a1aa7131371035be991/certifi-2026.5.20-py3-none-any.whl", hash = "sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897", size = 134134, upload-time = "2026-05-20T11:46:48.578Z" }, +] + +[[package]] +name = "click" +version = "8.4.1" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/9b/98/518d8e5081007684232226f475082b30087d0f585e8457db087298259f49/click-8.4.1.tar.gz", hash = "sha256:918b5633eddf6b41c32d4f454bf0de810065c74e3f7dbf8ee5452f8be88d3e96", size = 353007, upload-time = "2026-05-22T04:08:37.769Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c7/0d/67e5b4109ea4a837e80daa87c2c696711955e40449a97e8926672534def2/click-8.4.1-py3-none-any.whl", hash = "sha256:482be17c6991b8c19c5429a1e995d9b0efdbb63172824c41f99965dc0ade8ec2", size = 116639, upload-time = "2026-05-22T04:08:35.26Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "fastapi" +version = "0.140.0" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/0d/fb/fd7671137d9fa3df1d93a2f5111eb982709201724b29f211e4beb2d58688/fastapi-0.140.0.tar.gz", hash = "sha256:f338951b82fd74ca8f843163aec43ea1a1ce84d515415a50fa98fa25572a5544", size = 420968, upload-time = "2026-07-24T21:16:41.187Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/eb/76/6d9e25ad88da9d3ff744bcdbec4736e38c2288611d43f673a5d9bfa27c07/fastapi-0.140.0-py3-none-any.whl", hash = "sha256:e951c0a0d9540bf5d9a2a9e078fd415da2ab7e312d435139e7d9e2e7fe9f0b23", size = 130863, upload-time = "2026-07-24T21:16:42.89Z" }, +] + +[[package]] +name = "filelock" +version = "3.29.1" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/1f/f9/f38573ed5844586db374d085911740a501ccfa373b455fc9413f09f85237/filelock-3.29.1.tar.gz", hash = "sha256:d97e6b1b9757569626c58caa07dc4beb1613f4a2938b1e8cc81afca398906c9e", size = 59335, upload-time = "2026-06-03T15:19:04.053Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/4c/a0/614c5fe402fd88951df45f4dda2fa3b4e17a99ecd92340771929169b3b95/filelock-3.29.1-py3-none-any.whl", hash = "sha256:85199dfd706869641b72b2e8955d5416a4b2b7dc4b0e8e6d97b4cc1299a6983b", size = 40750, upload-time = "2026-06-03T15:19:02.959Z" }, +] + +[[package]] +name = "fsspec" +version = "2026.4.0" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d5/8d/1c51c094345df128ca4a990d633fe1a0ff28726c9e6b3c41ba65087bba1d/fsspec-2026.4.0.tar.gz", hash = "sha256:301d8ac70ae90ef3ad05dcf94d6c3754a097f9b5fe4667d2787aa359ec7df7e4", size = 312760, upload-time = "2026-04-29T20:42:38.635Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl", hash = "sha256:11ef7bb35dab8a394fde6e608221d5cf3e8499401c249bebaeaad760a1a8dec2", size = 203402, upload-time = "2026-04-29T20:42:36.842Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "hf-xet" +version = "1.5.0" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/74/d8/5c06fc76461418326a7decf8367480c35be11a41fd938633929c60a9ec6b/hf_xet-1.5.0.tar.gz", hash = "sha256:e0fb0a34d9f406eed88233e829a67ec016bec5af19e480eac65a233ea289a948", size = 837196, upload-time = "2026-05-06T06:18:15.583Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/68/9b/6912c99070915a4f28119e3c5b52a9abd1eec0ad5cb293b8c967a0c6f5a2/hf_xet-1.5.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:7d70fe2ce97b9db73b9c9b9c81fe3693640aec83416a966c446afea54acfae3c", size = 4023383, upload-time = "2026-05-06T06:17:53.947Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/0f/6d/9563cfde59b5d8128a9c7ec972a087f4c782e4f7bac5a85234edfd5d5e49/hf_xet-1.5.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:73a0dae8c71de3b0633a45c73f4a4a5ed09e94b43441d82981a781d4f12baa42", size = 3792751, upload-time = "2026-05-06T06:17:51.791Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/07/a5/ed5a0cf35b49a0571af5a8f53416dad1877a718c021c9937c3a53cb45781/hf_xet-1.5.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a60290ec57e9b71767fba7c3645ddafdd0759974b540441510c629c6db6db24a", size = 4456058, upload-time = "2026-05-06T06:17:40.735Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/60/fb/3ae8bf2a7a37a4197d0195d7247fd25b3952e15cb8a599e285dfaa6f52b3/hf_xet-1.5.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:e5de0f6deada0dada870bb376a11bcd1f08abf3a968a6d118f33e72d1b1eb480", size = 4250783, upload-time = "2026-05-06T06:17:38.412Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/a2/9b/8bae40d4d91525085137196e84eb0ed49cf65b5e96e5c3ecdadd8bd0fac2/hf_xet-1.5.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c799d49f1a5544a0ef7591c0ee75e0d6b93d6f56dc7a4979f59f7518d2872216", size = 4445594, upload-time = "2026-05-06T06:18:04.219Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/13/59/c74efbbd4e8728172b2cc72a2bc014d2947a4b7bdced932fbd3f5da1a4e5/hf_xet-1.5.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2baea1b0b989e5c152fe81425f7745ddc8901280ba3d97c98d8cdece7b706c60", size = 4663995, upload-time = "2026-05-06T06:18:06.1Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/73/32/8e1e0410af64cda9b139d1dcebdc993a8ff9c8c7c0e2696ae356d75ccc0d/hf_xet-1.5.0-cp313-cp313t-win_amd64.whl", hash = "sha256:526345b3ed45f374f6317349df489167606736c876241ba984105afe7fd4839d", size = 3966608, upload-time = "2026-05-06T06:18:19.74Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/fc/34/a8febc8f4edbea8b3e21b02ebc8b628679b84ba7e45cde624a7736b51500/hf_xet-1.5.0-cp313-cp313t-win_arm64.whl", hash = "sha256:786d28e2eb8315d5035544b9d137b4a842d600c434bb91bf7d0d953cce906ad4", size = 3796946, upload-time = "2026-05-06T06:18:17.568Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/2a/20/8fc8996afe5815fa1a6be8e9e5c02f24500f409d599e905800d498a4e14d/hf_xet-1.5.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:872d5601e6deea30d15865ede55d29eac6daf5a534ab417b99b6ef6b076dd96c", size = 4023495, upload-time = "2026-05-06T06:18:01.94Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/32/6a/93d84463c00cecb561a7508aa6303e35ee2894294eac14245526924415fe/hf_xet-1.5.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9929561f5abf4581c8ea79587881dfef6b8abb2a0d8a51915936fc2a614f4e73", size = 3792731, upload-time = "2026-05-06T06:18:00.021Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/9d/5a/8ec8e0c863b382d00b3c2e2af6ded6b06371be617144a625903a6d562f4b/hf_xet-1.5.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f7b7bbae318e583a86fb21e5a4a175d6721d628a2874f4bd022d0e660c32a682", size = 4456738, upload-time = "2026-05-06T06:17:49.574Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c5/ca/f7effa1a67717da2bcc6b6c28f71c6ca648c77acaec4e2c32f40cbe16d85/hf_xet-1.5.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:cf7b2dc6f31a4ea754bb50f74cde482dcf5d366d184076d8530b9872787f3761", size = 4251622, upload-time = "2026-05-06T06:17:47.096Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/65/f2/19247dba3e231cf77dec59ddfb878f00057635ff773d099c9b59d37812c3/hf_xet-1.5.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8dbcbab554c9ef158ef2c991545c3e970ddd8cc7acdcd0a78c5a41095dab4ded", size = 4445667, upload-time = "2026-05-06T06:18:11.983Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/7f/64/6f116801a3bcfb6f59f5c251f48cadc47ea54026441c4a385079286a94fa/hf_xet-1.5.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5906bf7718d3636dc13402914736abe723492cb730f744834f5f5b67d3a12702", size = 4664619, upload-time = "2026-05-06T06:18:13.771Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/5c/e8/069542d37946ed08669b127e1496fa99e78196d71de8d41eda5e9f1b7a58/hf_xet-1.5.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5f3dc2248fc01cc0a00cd392ab497f1ca373fcbc7e3f2da1f452480b384e839e", size = 3966802, upload-time = "2026-05-06T06:18:28.162Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f9/91/fc6fdec27b14d04e88c386ac0a0129732b53fa23f7c4a78f4b83a039c567/hf_xet-1.5.0-cp314-cp314t-win_arm64.whl", hash = "sha256:b285cea1b5bab46b758772716ba8d6854a1a0310fed1c249d678a8b38601e5a0", size = 3797168, upload-time = "2026-05-06T06:18:26.287Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/3d/fb/69ff198a82cae7eb1a69fb84d93b3a3e4816564d76817fe541ddc96874eb/hf_xet-1.5.0-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:dad0dc84e941b8ba3c860659fe1fdc35c049d47cce293f003287757e971a8f56", size = 4030814, upload-time = "2026-05-06T06:17:57.933Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/9b/ff/edcc2b40162bef3ff78e14ab637e5f3b89243d6aee72f5949d3bb6a5af83/hf_xet-1.5.0-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:fd6e5a9b0fdac4ed03ed45ef79254a655b1aaab514a02202617fbf643f5fdf7a", size = 3798444, upload-time = "2026-05-06T06:17:55.79Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/49/4d/103f76b04310e5e57656696cc184690d20c466af0bca3ca88f8c8ea5d4f3/hf_xet-1.5.0-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3531b1823a0e6d77d80f9ed15ca0e00f0d115094f8ac033d5cae88f4564cc949", size = 4465986, upload-time = "2026-05-06T06:17:44.886Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c4/a2/546f47f464737b3edbab6f8ddb57f2599b93d2cbb66f06abb475ccb48651/hf_xet-1.5.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9a0ee58cd18d5ea799f7ed11290bbccbe56bdd8b1d97ca74b9cc49a3945d7a3b", size = 4259865, upload-time = "2026-05-06T06:17:42.639Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/95/7f/1be593c1f28613be2e196473481cd81bfc5910795e30a34e8f744f6cac4f/hf_xet-1.5.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1e60df5a42e9bed8628b6416af2cba4cba57ae9f02de226a06b020d98e1aab18", size = 4459835, upload-time = "2026-05-06T06:18:08.026Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/aa/b2/703569fc881f3284487e68cda7b42179978480da3c438042a6bbbb4a671c/hf_xet-1.5.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4b35549ce62601b84da4ff9b24d970032ace3d4430f52d91bcbb26c901d6c690", size = 4672414, upload-time = "2026-05-06T06:18:09.864Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/af/37/1b6def445c567286b50aa3b33828158e135b1be44938dde59f11382a500c/hf_xet-1.5.0-cp37-abi3-win_amd64.whl", hash = "sha256:2806c7c17b4d23f8d88f7c4814f838c3b6150773fe339c20af23e1cfaf2797e4", size = 3977238, upload-time = "2026-05-06T06:18:23.621Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/62/94/3b66b148778ee100dcfd69c2ca22b57b41b44d3063ceec934f209e9184ce/hf_xet-1.5.0-cp37-abi3-win_arm64.whl", hash = "sha256:b6c9df403040248c76d808d3e047d64db2d923bae593eb244c41e425cf6cd7be", size = 3806916, upload-time = "2026-05-06T06:18:21.7Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "huggingface-hub" +version = "1.18.0" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "click" }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "httpx" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "tqdm" }, + { name = "typer" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/fb/d8/748ea0a47f0fa15227fe682f7a80826b4b7c096e4818044b8f56d6cb66d6/huggingface_hub-1.18.0.tar.gz", hash = "sha256:f0c5ecd1ef8c6a60f86f61ee278f2c1570ba9e279c9f54de9094210723b3613b", size = 812699, upload-time = "2026-06-05T09:26:33.401Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/0b/03/40a05316cb6616e5b7efd7773656441ab04b4b022c2199e79bb4622a92a3/huggingface_hub-1.18.0-py3-none-any.whl", hash = "sha256:729be4a976fb706dcc02d176bcda8a3f32bdf21a294e8f4b3dda6fbcbc9c1ab1", size = 684411, upload-time = "2026-06-05T09:26:31.48Z" }, +] + +[[package]] +name = "idna" +version = "3.18" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + +[[package]] +name = "linkify-it-py" +version = "2.1.0" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "uc-micro-py" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/2e/c9/06ea13676ef354f0af6169587ae292d3e2406e212876a413bf9eece4eb23/linkify_it_py-2.1.0.tar.gz", hash = "sha256:43360231720999c10e9328dc3691160e27a718e280673d444c38d7d3aaa3b98b", size = 29158, upload-time = "2026-03-01T07:48:47.683Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b4/de/88b3be5c31b22333b3ca2f6ff1de4e863d8fe45aaea7485f591970ec1d3e/linkify_it_py-2.1.0-py3-none-any.whl", hash = "sha256:0d252c1594ecba2ecedc444053db5d3a9b7ec1b0dd929c8f1d74dce89f86c05e", size = 19878, upload-time = "2026-03-01T07:48:46.098Z" }, +] + +[[package]] +name = "markdown-it-py" +version = "4.2.0" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, +] + +[package.optional-dependencies] +linkify = [ + { name = "linkify-it-py" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/08/db/fefacb2136439fc8dd20e797950e749aa1f4997ed584c62cfb8ef7c2be0e/markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad", size = 11631, upload-time = "2025-09-27T18:36:18.185Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/e1/2e/5898933336b61975ce9dc04decbc0a7f2fee78c30353c5efba7f2d6ff27a/markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a", size = 12058, upload-time = "2025-09-27T18:36:19.444Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/1d/09/adf2df3699d87d1d8184038df46a9c80d78c0148492323f4693df54e17bb/markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50", size = 24287, upload-time = "2025-09-27T18:36:20.768Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/30/ac/0273f6fcb5f42e314c6d8cd99effae6a5354604d461b8d392b5ec9530a54/markupsafe-3.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf", size = 22940, upload-time = "2025-09-27T18:36:22.249Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/19/ae/31c1be199ef767124c042c6c3e904da327a2f7f0cd63a0337e1eca2967a8/markupsafe-3.0.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f", size = 21887, upload-time = "2025-09-27T18:36:23.535Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b2/76/7edcab99d5349a4532a459e1fe64f0b0467a3365056ae550d3bcf3f79e1e/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a", size = 23692, upload-time = "2025-09-27T18:36:24.823Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/a4/28/6e74cdd26d7514849143d69f0bf2399f929c37dc2b31e6829fd2045b2765/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115", size = 21471, upload-time = "2025-09-27T18:36:25.95Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/62/7e/a145f36a5c2945673e590850a6f8014318d5577ed7e5920a4b3448e0865d/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a", size = 22923, upload-time = "2025-09-27T18:36:27.109Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/0f/62/d9c46a7f5c9adbeeeda52f5b8d802e1094e9717705a645efc71b0913a0a8/markupsafe-3.0.3-cp311-cp311-win32.whl", hash = "sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19", size = 14572, upload-time = "2025-09-27T18:36:28.045Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/83/8a/4414c03d3f891739326e1783338e48fb49781cc915b2e0ee052aa490d586/markupsafe-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01", size = 15077, upload-time = "2025-09-27T18:36:29.025Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/35/73/893072b42e6862f319b5207adc9ae06070f095b358655f077f69a35601f0/markupsafe-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c", size = 13876, upload-time = "2025-09-27T18:36:29.954Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622, upload-time = "2025-09-27T18:36:41.777Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029, upload-time = "2025-09-27T18:36:43.257Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543, upload-time = "2025-09-27T18:36:51.584Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113, upload-time = "2025-09-27T18:36:52.537Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911, upload-time = "2025-09-27T18:36:53.513Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658, upload-time = "2025-09-27T18:36:54.819Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066, upload-time = "2025-09-27T18:36:55.714Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612, upload-time = "2025-09-27T18:37:02.639Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200, upload-time = "2025-09-27T18:37:03.582Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, +] + +[[package]] +name = "mdit-py-plugins" +version = "0.6.1" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "markdown-it-py" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/59/fc/f8d0863f8862f25602c0404d75568e89fb6b4109804645e5cdfb1be5cf56/mdit_py_plugins-0.6.1.tar.gz", hash = "sha256:a2bca0f039f39dbd35fb74ae1b5f998608c437463371f0ff7f49a19a17a114d0", size = 56114, upload-time = "2026-05-13T09:03:38.91Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/a5/69/6da5581c6a7fede7dc261bf4e67d6adca4196f176b43288b55b3db395b6e/mdit_py_plugins-0.6.1-py3-none-any.whl", hash = "sha256:214c82fb2ac524472ab6a5bcab1de80f73b50443e187f401bfd77efbc7c6481d", size = 66663, upload-time = "2026-05-13T09:03:37.76Z" }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + +[[package]] +name = "mlx" +version = "0.31.2" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "mlx-metal", marker = "sys_platform == 'darwin'" }, +] +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/94/89/1e77ec3ff380e8fb9e7258047374d31452a0f9828a0e370f127b07dd8288/mlx-0.31.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4a3f181b367d404e44a6bd68ef5eb573930809ac60cacd51d0c851c629b1b651", size = 586911, upload-time = "2026-04-22T03:14:29.675Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/6a/41/c1907f05f8a3fc54025fb78ad68d3c4a4b931664d03c0a24f7f431cc4087/mlx-0.31.2-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:70297cbef7479429f69c966bfed10da20a6f0c2aa997eec2b4f6ba1a07caf2ef", size = 586915, upload-time = "2026-04-22T03:14:31.403Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/97/b0/61ac2c14773c786fecbda28067b0207a0c654cb4d10c548808c51284d700/mlx-0.31.2-cp311-cp311-macosx_26_0_arm64.whl", hash = "sha256:c0ff158b7ac93a4b5659adbc70053498b30a5964fc45f78596398e056a96c36a", size = 587030, upload-time = "2026-04-22T03:14:32.961Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/de/53/e12feb7078ee472983555fcb1da4749a2bbbc8fc5b29b78c205b96d37d1e/mlx-0.31.2-cp311-cp311-manylinux_2_35_aarch64.whl", hash = "sha256:cd5d42b0b2bee7efe1b0680a7e302943dd33b92c879cffa0358ffdb5a4a8d27b", size = 652994, upload-time = "2026-04-22T03:14:34.691Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c5/40/f92c8cdc9595bf24c7e483a3156bfe0cc99a5cf5545d8dba8e7fe000c10b/mlx-0.31.2-cp311-cp311-manylinux_2_35_x86_64.whl", hash = "sha256:b368f7ede4238cc44076e4843820338c453c21ee50bd3ee26d4b182c179fd8e1", size = 692086, upload-time = "2026-04-22T03:14:36.45Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c3/47/5f33906cb03d6a378a697cd2d2641a26b37dea17ee3d9124d7e39e8eca01/mlx-0.31.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:e5067aaf2be1f3d7bba5be52348775804f111173c1ed04639618fd713b1a530f", size = 584863, upload-time = "2026-04-22T03:14:38.211Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/08/e7/a851a451b1327af9fb4df3991b9ae87d066b6f6630e854af55c288b0995a/mlx-0.31.2-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:edb9797db7d852477ca1c99708058654ee860d4148fe5765f0d55528e2b1aa22", size = 584860, upload-time = "2026-04-22T03:14:39.746Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/3b/15/0d1dc0597644e5e7b011ca954ba0c47e13cd880a3b909b0c3f1b4d8bf8f1/mlx-0.31.2-cp312-cp312-macosx_26_0_arm64.whl", hash = "sha256:51ca102db641b01e7cb083ce8ecb580e281530a141a7ca12544bb370641630ae", size = 584887, upload-time = "2026-04-22T03:14:41.585Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/5d/c3/00664239a98e8bd614733c4182cd402d2bacad2d7f79eca66562ac406870/mlx-0.31.2-cp312-cp312-manylinux_2_35_aarch64.whl", hash = "sha256:117c7583cae0ca107cd53c591cc34f8e75f97a505aa47088844b7dc0fc69dc67", size = 627863, upload-time = "2026-04-22T03:14:43.326Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/53/7b/af6cd73a79772af6f19eab2cb4c48eda23a9294d1650a4c1269a9996e532/mlx-0.31.2-cp312-cp312-manylinux_2_35_x86_64.whl", hash = "sha256:99572133181481640a8bf8d449daf083816d0af3ee050c8adfc5bf45ceca91c6", size = 685090, upload-time = "2026-04-22T03:14:45.058Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/a3/3f/888f8664d4f8e23a1363a5f50024be5216e199ab7ad0ba20988c7ed6d729/mlx-0.31.2-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:1b3fb0dda955b0d552ce57bdd6f42b3309ab21b067e40587d6848443d307e91f", size = 584796, upload-time = "2026-04-22T03:14:47.215Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/dd/14/e9cd18b51f9e1dbcb060eec0fafc2d2428c8e1eacd9b0a02d7c5ce75b661/mlx-0.31.2-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:34b0171cd9eb5c43fdd82091f6135d6ccc5a065363a4a3e68fac64fb4e53d37c", size = 584790, upload-time = "2026-04-22T03:14:48.519Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ca/20/c6c5fb998c7834d094b2bfb9f003b5246cb270f0266da055c55546c34999/mlx-0.31.2-cp313-cp313-macosx_26_0_arm64.whl", hash = "sha256:c05981684279a8935d58b0dde3ea5b02d210c3bad3319aa0e9934ec2df165752", size = 584795, upload-time = "2026-04-22T03:14:49.904Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/0b/19/aca251d4c5f3532ce9c2c1e95ad76740d9c6c298f406f62d992f465b9be0/mlx-0.31.2-cp313-cp313-manylinux_2_35_aarch64.whl", hash = "sha256:cd1f4189e5f1bc68735f44eb63ce98ae09d66ac75d7ab5b15a41afae7e9f0513", size = 627843, upload-time = "2026-04-22T03:14:51.351Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/3e/2b/b89364883b98f21c2fe29e52d4ac8bc2fa2fe0d79293b36ec421efc1854a/mlx-0.31.2-cp313-cp313-manylinux_2_35_x86_64.whl", hash = "sha256:53c8d57ffa9ce77f8355663be05014c0dd37280e57f19126fb0a24389a30684b", size = 685064, upload-time = "2026-04-22T03:14:52.75Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/8b/f5/e63f6a9316ded2d14a8ebc7a9ca25734c784e8c54d064a78b4dceeacec0e/mlx-0.31.2-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:a13c9ce23c3deef6aa5a09315e7953e1a5dc311e851fa16fc74c81fb2509c0b9", size = 588417, upload-time = "2026-04-22T03:14:54.094Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/31/50/9d0c03ea3134cd85c132df7b0e4b75e6344bd8b4881a0b9c465cfa27f724/mlx-0.31.2-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:b0764bf11fc3a71dee988e19275eef67775cab63112d8bb7ef173ca8b2a1247c", size = 588421, upload-time = "2026-04-22T03:14:55.898Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ef/5b/d364cc793bcb504621313acb55627cf0d5403ab2e0a594aa081cdbe4591f/mlx-0.31.2-cp314-cp314-macosx_26_0_arm64.whl", hash = "sha256:59ccbd0f0044d4f97f11ebcbf0c480bc9e962935fd96275f120954afea65be8a", size = 588384, upload-time = "2026-04-22T03:14:57.439Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/1b/a4/e822202dd2e4e7d08671f2ecf7b6500af74f5bad5ceb27086b1aa6902f3a/mlx-0.31.2-cp314-cp314-manylinux_2_35_aarch64.whl", hash = "sha256:e81798c610f95a09c642c89214ba5c23b72ce18ce4728184aceabe7eddca33d7", size = 630473, upload-time = "2026-04-22T03:14:58.985Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/40/6f/da48d2d7a76e644d35438ef6f33c68755fdd382e2c546fd1804ccba01d04/mlx-0.31.2-cp314-cp314-manylinux_2_35_x86_64.whl", hash = "sha256:69fbc94bf53607a75af9eb3e22c354738a6fe4e25aa4e2b20934b009a4bba1f3", size = 685459, upload-time = "2026-04-22T03:15:00.45Z" }, +] + +[[package]] +name = "mlx-lm" +version = "0.31.3" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "jinja2" }, + { name = "mlx", marker = "sys_platform == 'darwin'" }, + { name = "numpy" }, + { name = "protobuf" }, + { name = "pyyaml" }, + { name = "sentencepiece" }, + { name = "transformers" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/84/94/9a38d6b0c6fcca995b9136c94eb7da1e9c5165652edf228b96b29960fa7a/mlx_lm-0.31.3.tar.gz", hash = "sha256:61eb0e3ba09444f77f874aff295401d7ccd20b39495cbbce0c782a15474ce733", size = 304318, upload-time = "2026-04-22T07:37:27.922Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/90/02/9a67b8e4f87e3e2e5cd7b1ad79304b93c09a0db6af34bee75e6551c06c60/mlx_lm-0.31.3-py3-none-any.whl", hash = "sha256:758cfddf1180053b7613db76fad3d246a331a2a905808e1164a275621fc983b8", size = 408890, upload-time = "2026-04-22T07:37:25.965Z" }, +] + +[[package]] +name = "mlx-metal" +version = "0.31.2" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/3f/69/fe3b783ebe999f3118234e1e940feb622518bfb1dea6ac5d13b1d36a8449/mlx_metal-0.31.2-py3-none-macosx_14_0_arm64.whl", hash = "sha256:b25385bcee18fc194092255b8b53b9a3d8489eb650e59160f1b57aadd07aa2dc", size = 40055588, upload-time = "2026-04-22T03:14:14.43Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/4f/5d/4c690d5b93c30ba002656c37363159d978705bf8eb801b8481840fb942c2/mlx_metal-0.31.2-py3-none-macosx_15_0_arm64.whl", hash = "sha256:e9d4e5fce6ca10a87a0e388597f99519ad594d09e674708b5312bd8bd4f5997d", size = 40053220, upload-time = "2026-04-22T03:14:18.048Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/99/82/11fd62a8d7a3e96e5c43220b17de0151e3f10101f8bb3b865f5bd9cdd074/mlx_metal-0.31.2-py3-none-macosx_26_0_arm64.whl", hash = "sha256:84ffb60ee503f03eb684f5fb168d5cff31e2a16b7f27c1731eaf7662bd6e9b46", size = 55792151, upload-time = "2026-04-22T03:14:22.059Z" }, +] + +[[package]] +name = "nanobind" +version = "2.12.0" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c6/5c/3b69bc3933ad3c3668ba029ad410ba8ecfdc8ee7262ff1009f3304f3c562/nanobind-2.12.0.tar.gz", hash = "sha256:0ae77c1a88f27153fa57045ee00f7b0a7b06b1cd3df942e95a34b38c5d0a5bee", size = 1002704, upload-time = "2026-02-25T09:41:54.691Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/5a/bf/1a54e3573736f3ad15fc599c5dde007937234652a1a7efd62573b4ce3a7e/nanobind-2.12.0-py3-none-any.whl", hash = "sha256:a10d3d88e691dcdf22696f9acd893fda3c5a05635763aea238829d274fcad480", size = 249512, upload-time = "2026-02-25T09:41:52.908Z" }, +] + +[[package]] +name = "numpy" +version = "2.4.6" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d0/ad/fed0499ce6a338d2a03ebae59cd15093910c8875328855781952abf6c2fe/numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda", size = 20735807, upload-time = "2026-05-18T23:37:14.07Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b3/49/ec46835a70be8fa6446c495126ac84fdb28cb2558e1620ffb87a10c8b64c/numpy-2.4.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4", size = 16969194, upload-time = "2026-05-18T23:33:13.503Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/0e/0d/f5957185c0ee2f3e12f78715aa9e3b353fd83633316c8532b38faa37e3f6/numpy-2.4.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d", size = 14964111, upload-time = "2026-05-18T23:33:17.795Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ad/40/40a40ee0ddf7ceb782c49af278894b686e586d65d8c1889c8b5da01a3d7d/numpy-2.4.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8", size = 5469159, upload-time = "2026-05-18T23:33:20.654Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/63/13/f9a8046535cb21deae82f8d03de9617e08882d274fad2539630761888228/numpy-2.4.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538", size = 6798936, upload-time = "2026-05-18T23:33:22.987Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/33/a8/6fa8c1a345a8c85dbb21932c447bee07c30a2c2a3f31e369c0a84b300147/numpy-2.4.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47", size = 15966692, upload-time = "2026-05-18T23:33:26.62Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/02/03/74fe2a4cb3817d94d86402f2506554130a2f01414e299b5a843e5a8a957f/numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93", size = 16918164, upload-time = "2026-05-18T23:33:29.955Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c5/80/3615be3313f7e7696609bc194b9f0101da809df79e859bdb84e0cd043f46/numpy-2.4.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8", size = 17322877, upload-time = "2026-05-18T23:33:34.724Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ca/ac/a691e0fe2675e370d0e08ff905adc49a1c8830e8cae03efe4477e92cd55d/numpy-2.4.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6", size = 18651487, upload-time = "2026-05-18T23:33:38.217Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/15/a7/9bc1cd626d7bf6869bfedf27b91b6ab5dd607758bf8e959d6fa80c6a59cb/numpy-2.4.6-cp311-cp311-win32.whl", hash = "sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8", size = 6233945, upload-time = "2026-05-18T23:33:41.331Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c5/31/7fc6239c12bce7e931463251cca4426c465e1876ba3cc785402ef4dd8f4e/numpy-2.4.6-cp311-cp311-win_amd64.whl", hash = "sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147", size = 12608406, upload-time = "2026-05-18T23:33:44.131Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/27/83/140f85a466595a16382996a1bf06b2b54bcd597488921b0c9daaeeda72af/numpy-2.4.6-cp311-cp311-win_arm64.whl", hash = "sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577", size = 10479528, upload-time = "2026-05-18T23:33:50.725Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/95/2a/3d7b5ac8aac24feaf9ad7ed58f45b0bbc06d37e4338ae84c9f2298b570f9/numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1", size = 16689119, upload-time = "2026-05-18T23:33:54.065Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ea/12/92c4c131527599e8288d6918e888d88726f84d805d784b771f32408aeaef/numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb", size = 14699246, upload-time = "2026-05-18T23:33:57.621Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ad/fe/c0a6b7b2ca128a8fb228575147073b660656734b8ebe4d76c8fd748dcc79/numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41", size = 5204410, upload-time = "2026-05-18T23:34:00.302Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f3/d4/9770d14ba719432bb90a421bfd443872ed0f70f7264b64bec12ea363d5fd/numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698", size = 6551240, upload-time = "2026-05-18T23:34:02.852Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c9/c6/50a46a6205feba2343f1d6d17438107c5dc491ed1c736e6ea68689fd906b/numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f", size = 15671012, upload-time = "2026-05-18T23:34:05.485Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/99/60/14115e6364fa676c5397c2ad3004e527e9aa487abf5d0706ec81bbd08529/numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853", size = 16645538, upload-time = "2026-05-18T23:34:09.265Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ae/c5/693cbe59e57db94d2231fa519ca3978dc9e19da5a8f088588f5c6e947ff2/numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a", size = 17020706, upload-time = "2026-05-18T23:34:13.053Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ef/fc/85b7c4eff9b4966ade25c2273cf7e7012e92366c032058653934b37de044/numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2", size = 18368541, upload-time = "2026-05-18T23:34:17.024Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f6/81/e1b27545deedce7f4a0b348618c6b62d74e36a4dc9ccd42f3eb2f85eee32/numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45", size = 5962825, upload-time = "2026-05-18T23:34:20.3Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ab/ca/feab00bd44aa5fe1ad2c18f08b4d3bb92e26484b0b1d1443897809ed528c/numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751", size = 12321687, upload-time = "2026-05-18T23:34:23.095Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/63/cf/5a6d34850a39d1093558564f77ee8e8e0bee5061151b8f05a55711001ec7/numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8", size = 10221482, upload-time = "2026-05-18T23:34:25.876Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/fb/82/bdab26d7438c6791ca31b7c024ca37c1eab8b726ba236129005cd4a06e45/numpy-2.4.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0", size = 16684648, upload-time = "2026-05-18T23:34:29.41Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/1b/30/a80189bcc7f5e4258b3fbc3968d909d1756f54d023299ecc39ad6fdb9ef8/numpy-2.4.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb", size = 14693902, upload-time = "2026-05-18T23:34:33.013Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/97/12/70b5d0d7c15e1ebb8a6a84a8caa1d19e181d84fb58bb6d70aca29099dec1/numpy-2.4.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f", size = 5198992, upload-time = "2026-05-18T23:34:36.132Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ba/8c/ebd2a8f8a83541f8d38cc5667e8c2b69cecfd30da6e45693e8158857d44b/numpy-2.4.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3", size = 6546944, upload-time = "2026-05-18T23:34:38.484Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/bb/c5/7b863a97a91671a0338f4253bd3b5a3d3852f0692dae91711c9f4a10e787/numpy-2.4.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b", size = 15669392, upload-time = "2026-05-18T23:34:41.257Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/a5/9d/3584b9984ca4c047aea75214ce1a4c4c73d849bd71b604264b7f5653f8a8/numpy-2.4.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089", size = 16633220, upload-time = "2026-05-18T23:34:45.075Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/05/ae/7c67fba23bd98caec7c99261f3a16072ade14813486b0282cb29846de832/numpy-2.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a", size = 17020800, upload-time = "2026-05-18T23:34:49.065Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d9/5d/3b6725cb31d983c5e66916f5d36f6d7e5521129e4c4404d64f918292a5b6/numpy-2.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605", size = 18357600, upload-time = "2026-05-18T23:34:52.709Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f7/da/2ccc6c2fe8898dee01d90c75c5f5f914a23daf99e3e0f59516a08760c8b5/numpy-2.4.6-cp313-cp313-win32.whl", hash = "sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91", size = 5961134, upload-time = "2026-05-18T23:34:55.618Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b5/cd/9cc4dc876fb065d5c220aae4d5e14826b2715331bb7618ce1fb07a679d99/numpy-2.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359", size = 12318598, upload-time = "2026-05-18T23:34:58.928Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/39/1e/c0bcba1f8694116485fe28fd1be698c278fcda4141c5b0e53a2aed8b12a8/numpy-2.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778", size = 10222272, upload-time = "2026-05-18T23:35:02.167Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/63/6d/cc5619247c8f4204e507f5883528372e4ac4bb189e579fb859a12e480b1f/numpy-2.4.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1", size = 14821197, upload-time = "2026-05-18T23:35:05.468Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/00/58/f1c39161c87d9e9bed660f1ed4bafc0e403d5ec9650b6dd77aead07d489b/numpy-2.4.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe", size = 5326287, upload-time = "2026-05-18T23:35:08.693Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/af/57/3917ab0fd97f271a8694513581b8a36c655f111c446852c302f04ccdb6fc/numpy-2.4.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997", size = 6646763, upload-time = "2026-05-18T23:35:11.459Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/eb/0f/037e64c494b67581ae18193d770adef354c41f3f2c8ebf865602d949bf8f/numpy-2.4.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20", size = 15728070, upload-time = "2026-05-18T23:35:14.79Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/21/a6/5d2bae9c9542eb4df16dc9c46dc79c186e9bad53805dfa5399a6023c6db0/numpy-2.4.6-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d", size = 16681752, upload-time = "2026-05-18T23:35:18.836Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/92/14/23d1dfb410ae362cd59ce53e936b1513d545eb40db3949ced632e19a459e/numpy-2.4.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67", size = 17086024, upload-time = "2026-05-18T23:35:22.52Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/4b/6e/23595a2c642cdf3bc567877064bdd7f91c8b0038a4453cf2daf7248eafe9/numpy-2.4.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd", size = 18403398, upload-time = "2026-05-18T23:35:26.398Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/8a/90/0ac3bc947217e66dec77e7cbc6a1979d1af70b6461b82f620d3bccd5e4c8/numpy-2.4.6-cp313-cp313t-win32.whl", hash = "sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab", size = 6084971, upload-time = "2026-05-18T23:35:29.387Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/77/71/5673e351671a1d2bd6063b91b44f70c0affea7d1516fa7a6572941ba4aa1/numpy-2.4.6-cp313-cp313t-win_amd64.whl", hash = "sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75", size = 12458532, upload-time = "2026-05-18T23:35:32.175Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/3f/88/19d3503c5046e688f049274b27a3ef3d771152fa80d3ba3d01a3dff61abe/numpy-2.4.6-cp313-cp313t-win_arm64.whl", hash = "sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd", size = 10291881, upload-time = "2026-05-18T23:35:35.465Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f8/91/3ab2044d05fd16d343c5ac2e69b127f1b2854040dd20b193257c78028bd3/numpy-2.4.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079", size = 16683458, upload-time = "2026-05-18T23:35:38.353Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/8e/62/764ce66fa4147ae6d73071a3abf804ffe606f174618697c571acdf26a7c9/numpy-2.4.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7", size = 14704559, upload-time = "2026-05-18T23:35:42.14Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/60/61/23f27c172f022e04025b7dc2367f4d63c1a398120607ec896228649a6f48/numpy-2.4.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5", size = 5209716, upload-time = "2026-05-18T23:35:45.377Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/03/71/21cf70dc6ea3e3acb95fc53a265b2fc248b981f0194ceb5b475271b8809d/numpy-2.4.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096", size = 6543947, upload-time = "2026-05-18T23:35:47.926Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d5/91/64288395ee1799bd2e0b04a305dce9666da90c961e1f3fe982a05ee1c036/numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b", size = 15685197, upload-time = "2026-05-18T23:35:50.863Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f3/eb/ebffaa97dc55502df69584a8f0dcf07f69a3e0b3e2323670a2722db9aa39/numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8", size = 16638245, upload-time = "2026-05-18T23:35:54.752Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b8/0b/54f9da33128d7e350fab89c7455902eeae70349ee52bddb448dc4a576f45/numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402", size = 17036587, upload-time = "2026-05-18T23:35:58.355Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b6/f0/fdebc1052db1cc37c64beb22072d67cd6d1c71adca1299f53dec2b5e20d3/numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb", size = 18363226, upload-time = "2026-05-18T23:36:02.845Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/aa/b4/298628d98c72b57e57f7165ae6a481a1deaf6f3c28262a6e4c739c275930/numpy-2.4.6-cp314-cp314-win32.whl", hash = "sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1", size = 6010196, upload-time = "2026-05-18T23:36:05.92Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/df/ac/46de6dda46478f7942f839e094970be2d4a861e005c4b3bf07c92e291a09/numpy-2.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261", size = 12450334, upload-time = "2026-05-18T23:36:09.107Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/78/92/b8b798ac784102c0da830d2257d59358e3d3d90d1e2b3f2575dad976c5cf/numpy-2.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6", size = 10495678, upload-time = "2026-05-18T23:36:12.766Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/30/34/ec28d1aa8115971537c01469ab2011ee96827930f0a124de1000cc2a7ed7/numpy-2.4.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a", size = 14823672, upload-time = "2026-05-18T23:36:16.473Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/16/bd/f6d1fede4e54e8042a7ff97bb495510f3c220f94bcd9e8b228e87c92cc0d/numpy-2.4.6-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e", size = 5328731, upload-time = "2026-05-18T23:36:19.767Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f4/f0/e105b9e2fd728a9910103884decd6951d9dd73896b914a98d9a231de02ee/numpy-2.4.6-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e", size = 6649805, upload-time = "2026-05-18T23:36:22.266Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/82/dd/1206a7ca6ab15e3f02069707ca96222e202af681bb73756da7527f3cb837/numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43", size = 15730496, upload-time = "2026-05-18T23:36:25.713Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/51/e7/38d3ea825dcab85a591734decb2f6c67caa7c8367d374df1a1c3842f9b07/numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e", size = 16679616, upload-time = "2026-05-18T23:36:29.652Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/93/b7/caabfdf53edf663e0b4eb74d7d405d83baef09eb5e83bcd32d601d72b93e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895", size = 17085145, upload-time = "2026-05-18T23:36:33.449Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f9/45/68d7c33a6bcf3e5aa3bdbd57a367e6f615286dfd6482f97e8ffeb734306e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4", size = 18403813, upload-time = "2026-05-18T23:36:37.369Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/9c/50/0753655aa844c99cd9e018aacf76f130f1bd81d881bb74bc0aef5d73a8ba/numpy-2.4.6-cp314-cp314t-win32.whl", hash = "sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063", size = 6156982, upload-time = "2026-05-18T23:36:40.817Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b2/d4/7c67becf668f973cb490cec3e98dfd799d866f9c989a54d355672cfa0db6/numpy-2.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627", size = 12638908, upload-time = "2026-05-18T23:36:43.996Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/43/bb/e1c71a4295b1b1d1393d50dbb4f2a36283c6859d9d3892e84f00ec5a91d5/numpy-2.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66", size = 10565867, upload-time = "2026-05-18T23:36:47.114Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/de/12/b422cc84439adc0d00de605bf4a308890ae5c26f2c71fbd73e5d08fbb0dd/numpy-2.4.6-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662", size = 16847511, upload-time = "2026-05-18T23:36:50.673Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/44/53/f481bef68011740f8849418d82db07230e825013f31f4eef5ba5b805316a/numpy-2.4.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7", size = 14889064, upload-time = "2026-05-18T23:36:53.879Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/7f/57/42ed575c10ced8af951d426bc4e1f8aff16fd851db33f067036215a7f860/numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f", size = 5394157, upload-time = "2026-05-18T23:36:57.194Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/6a/ef/f66cc724fcc36c1e364c67f51ae9146090b8b584f27d58b97fdae3edd737/numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c", size = 6708728, upload-time = "2026-05-18T23:36:59.575Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/1a/9c/c531f2293b91265d8b48e9b329f54fdd7ffae73cb4134ea10cca4237e9cc/numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0", size = 15798374, upload-time = "2026-05-18T23:37:02.674Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/1a/b0/413077f6b1153ed3cba361401c6783bbad6114804a000cc22eb71c13e190/numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02", size = 16747286, upload-time = "2026-05-18T23:37:06.327Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/15/ce/e5ec180bc41812edcd8daeb8639d205622c0e8c02259d8ab25a0201b3c2a/numpy-2.4.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73", size = 12504263, upload-time = "2026-05-18T23:37:09.715Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "platformdirs" +version = "4.10.0" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d7/47/e4501f49c178ae1d9f4a75073fda4204f52647993f075a9db4d14930e0c5/platformdirs-4.10.0.tar.gz", hash = "sha256:31e761a6a0ca04faf7353ea759bdba55652be214725111e5aac52dfa29d4bef7", size = 31224, upload-time = "2026-05-28T03:32:53.587Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/81/e6/cd9575ac904136b3cbf7aa7ee819ef86eedb7274e46f230e94ea4342e729/platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a", size = 22743, upload-time = "2026-05-28T03:32:52.175Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "protobuf" +version = "7.35.0" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/60/fd/5b1491d9e4b586d621c54f4c36b888714164b6875f8d6afa3f9072906a51/protobuf-7.35.0.tar.gz", hash = "sha256:a2efd84605f41e559f1881b0912b44099d0a2ac9bf46b3474823f10fb393b0e6", size = 458677, upload-time = "2026-05-19T23:02:29.197Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/83/ee/93d06e358a4aa32280b00e722d3ea0a1f25fc3cc5778d80581c9cca2c10e/protobuf-7.35.0-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:66be6c513931c794fa92c080ffee41671390da3d79da219cf9c0c0907f035dda", size = 433225, upload-time = "2026-05-19T23:02:19.884Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/8b/39/1c76c2da93f3c507e958e0aecee2391cc44d4625de6c728bbc555195b5a8/protobuf-7.35.0-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:fcbe42a4ac09d3ec9c987ddfcd956afd0b15f1ff613bd8371bde9405ffd5c8e5", size = 328847, upload-time = "2026-05-19T23:02:22.3Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/91/1a/39f7ce90a238c1a987a4d81ec26379e02ca0aff367de68e4a1fa474215b9/protobuf-7.35.0-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:4cbf5cc286130e06a6c9bbefac442431173906dfcc979712183d4adcc01b37ee", size = 344030, upload-time = "2026-05-19T23:02:23.591Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/70/5b/6baf9008817964454055ff3fe65f1de0b5f1e26c80c82f7fb108b7cd4ea3/protobuf-7.35.0-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:6c0f98f10c8a05ea30f8993dfef2de093d27b490fdae78bb60c8343795d55011", size = 327130, upload-time = "2026-05-19T23:02:24.637Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/8e/e5/e46adb0badc388bfb84877a5f9f026aff63f60e611016cf64dbe77e05446/protobuf-7.35.0-cp310-abi3-win32.whl", hash = "sha256:4c4617b83ade0e279d1d2bfe04025a1adb87f9ed657de038620dc0ff959357f6", size = 428946, upload-time = "2026-05-19T23:02:25.741Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/a7/ab/547fbd9e16d879dd13c167478f8ae0a83a428008ca07a5e06acdc23ad473/protobuf-7.35.0-cp310-abi3-win_amd64.whl", hash = "sha256:f05bcadf9a2a6b8dda047007075135fb7d08c73d9177aabc067e1be46881a201", size = 439996, upload-time = "2026-05-19T23:02:26.808Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b8/ef/50433d346c56657a70d27f156c7b349ac59a068b01de4eb796e747eecc43/protobuf-7.35.0-py3-none-any.whl", hash = "sha256:c13f325cf242bad135c350629eeb5d54b24228eb472fb3e2e9ebbd4c5dc20ca0", size = 171659, upload-time = "2026-05-19T23:02:27.842Z" }, +] + +[[package]] +name = "pydantic" +version = "2.13.4" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/5c/fa/6d7708d2cfc1a832acb6aeb0cd16e801902df8a0f583bb3b4b527fde022e/pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594", size = 2111872, upload-time = "2026-05-06T13:40:27.596Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ae/6f/aa064a3e74b5745afbdf250594f38e7ead05e2d651bcb35994b9417a0d4d/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c", size = 1948255, upload-time = "2026-05-06T13:39:12.574Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/43/3a/41114a9f7569b84b4d84e7a018c57c56347dac30c0d4a872946ec4e36c46/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826", size = 1972827, upload-time = "2026-05-06T13:38:19.841Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ef/25/1ab42e8048fe551934d9884e8d64daa7e990ad386f310a15981aeb6a5b08/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04", size = 2041051, upload-time = "2026-05-06T13:38:10.447Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/94/c2/1a934597ddf08da410385b3b7aae91956a5a76c635effef456074fad7e88/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e", size = 2221314, upload-time = "2026-05-06T13:40:13.089Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/02/6d/9e8ad178c9c4df27ad3c8f25d1fe2a7ab0d2ba0559fad4aee5d3d1f16771/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3", size = 2285146, upload-time = "2026-05-06T13:38:59.224Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/80/50/540cd3aeefc041beb111125c4bff779831a2111fc6b15a9138cda277d32c/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4", size = 2089685, upload-time = "2026-05-06T13:38:17.762Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/6b/a4/b440ad35f05f6a38f89fa0f149accb3f0e02be94ca5e15f3c449a61b4bc9/pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398", size = 2115420, upload-time = "2026-05-06T13:37:58.195Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/99/61/de4f55db8dfd57bfdfa9a12ec90fe1b57c4f41062f7ca86f08586b3e0ac0/pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3", size = 2165122, upload-time = "2026-05-06T13:37:01.167Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f7/52/7c529d7bdb2d1068bd52f51fe32572c8301f9a4febf1948f10639f1436f5/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848", size = 2182573, upload-time = "2026-05-06T13:38:45.04Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/37/b3/7c40325848ba78247f2812dcf9c7274e38cd801820ca6dd9fe63bcfb0eb4/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3", size = 2317139, upload-time = "2026-05-06T13:37:15.539Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d9/37/f913f81a657c865b75da6c0dbed79876073c2a43b5bd9edbe8da785e4d49/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109", size = 2360433, upload-time = "2026-05-06T13:37:30.099Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c4/67/6acaa1be2567f9256b056d8477158cac7240813956ce86e49deae8e173b4/pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda", size = 1985513, upload-time = "2026-05-06T13:38:15.669Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/aa/e6/c505f83dfeda9a2e5c995cfd872949e4d05e12f7feb3dca72f633daefa94/pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33", size = 2071114, upload-time = "2026-05-06T13:40:35.416Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/0f/da/7a263a96d965d9d0df5e8de8a475f33495451117035b09acb110288c381f/pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d", size = 2044298, upload-time = "2026-05-06T13:38:29.754Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158, upload-time = "2026-05-06T13:38:57.215Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724, upload-time = "2026-05-06T13:37:02.697Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418, upload-time = "2026-05-06T13:37:38.234Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274, upload-time = "2026-05-06T13:38:27.753Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940, upload-time = "2026-05-06T13:38:05.353Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516, upload-time = "2026-05-06T13:39:10.577Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854, upload-time = "2026-05-06T13:40:22.59Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306, upload-time = "2026-05-06T13:40:10.666Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044, upload-time = "2026-05-06T13:40:43.231Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133, upload-time = "2026-05-06T13:39:57.365Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464, upload-time = "2026-05-06T13:38:06.976Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", size = 1974823, upload-time = "2026-05-06T13:40:47.985Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", size = 2072919, upload-time = "2026-05-06T13:39:21.153Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", size = 2027604, upload-time = "2026-05-06T13:39:03.753Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/51/a2/5d30b469c5267a17b39dec53208222f76a8d351dfac4af661888c5aee77d/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", size = 2106306, upload-time = "2026-05-06T13:37:48.029Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c1/81/4fa520eaffa8bd7d1525e644cd6d39e7d60b1592bc5b516693c7340b50f1/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", size = 1951906, upload-time = "2026-05-06T13:37:17.012Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", size = 1976802, upload-time = "2026-05-06T13:37:35.113Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", size = 2052446, upload-time = "2026-05-06T13:37:12.313Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", size = 2232757, upload-time = "2026-05-06T13:39:01.149Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", size = 2309275, upload-time = "2026-05-06T13:37:41.406Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", size = 2094467, upload-time = "2026-05-06T13:39:18.847Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", size = 2134417, upload-time = "2026-05-06T13:40:17.944Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", size = 2179782, upload-time = "2026-05-06T13:40:32.618Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", size = 2188782, upload-time = "2026-05-06T13:36:51.018Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", size = 2328334, upload-time = "2026-05-06T13:40:37.764Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", size = 2372986, upload-time = "2026-05-06T13:39:34.152Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/5b/24/b375a480d53113860c299764bfe9f349a3dc9108b3adc0d7f0d786492ebf/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", size = 1973693, upload-time = "2026-05-06T13:37:55.072Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/7e/e8/cff247591966f2d22ec8c003cd7587e27b7ba7b81ab2fb888e3ab75dc285/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", size = 2071819, upload-time = "2026-05-06T13:38:49.139Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c6/1a/f4aee670d5670e9e148e0c82c7db98d780be566c6e6a97ee8035528ca0b3/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", size = 2027411, upload-time = "2026-05-06T13:40:45.796Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079, upload-time = "2026-05-06T13:38:41.019Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179, upload-time = "2026-05-06T13:36:59.812Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354, upload-time = "2026-05-06T13:38:03.499Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238, upload-time = "2026-05-06T13:39:40.807Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251, upload-time = "2026-05-06T13:37:26.72Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593, upload-time = "2026-05-06T13:39:47.682Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226, upload-time = "2026-05-06T13:40:40.428Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ee/a4/73995fd4ebbb46ba0ee51e6fa049b8f02c40daebb762208feda8a6b7894d/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c", size = 2111589, upload-time = "2026-05-06T13:37:10.817Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/fb/7f/f37d3a5e8bfcc2e403f5c57a730f2d815693fb42119e8ea48b3789335af1/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b", size = 1944552, upload-time = "2026-05-06T13:36:56.717Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/15/3c/d7eb777b3ff43e8433a4efb39a17aa8fd98a4ee8561a24a67ef5db07b2d6/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b", size = 1982984, upload-time = "2026-05-06T13:39:06.207Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/63/87/70b9f40170a81afd55ca26c9b2acb25c20d64bcfbf888fafecb3ba077d4c/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea", size = 2138417, upload-time = "2026-05-06T13:39:45.476Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527, upload-time = "2026-05-06T13:39:52.283Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024, upload-time = "2026-05-06T13:40:15.671Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/11/cb/428de0385b6c8d44b716feba566abfacfbd23ee3c4439faa789a1456242f/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0", size = 2112782, upload-time = "2026-05-06T13:37:04.016Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/0b/b5/6a17bdadd0fc1f170adfd05a20d37c832f52b117b4d9131da1f41bb097ce/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7", size = 1952146, upload-time = "2026-05-06T13:39:43.092Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/2a/dc/03734d80e362cd43ef65428e9de77c730ce7f2f11c60d2b1e1b39f0fbf99/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2", size = 2134492, upload-time = "2026-05-06T13:36:58.124Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/de/df/5e5ffc085ed07cc22d298134d3d911c63e91f6a0eb91fe646750a3209910/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9", size = 2156604, upload-time = "2026-05-06T13:37:49.88Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/81/44/6e112a4253e56f5705467cbab7ab5e91ee7398ba3d56d358635958893d3e/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf", size = 2183828, upload-time = "2026-05-06T13:37:43.053Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ac/ad/5565071e937d8e752842ac241463944c9eb14c87e2d269f2658a5bd05e98/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30", size = 2310000, upload-time = "2026-05-06T13:37:56.694Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/4f/c3/66883a5cec183e7fba4d024b4cbbe61851a63750ef606b0afecc46d1f2bf/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc", size = 2361286, upload-time = "2026-05-06T13:40:05.667Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.0.3" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, +] + +[[package]] +name = "pytest-asyncio" +version = "1.4.0" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "pytest" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/43/7c/d36d04db312ecf4298932ef77e6e4a9e8ad017906e24e34f0b0c361a2473/pytest_asyncio-1.4.0.tar.gz", hash = "sha256:c6c0d2259945122819f171a32ecea2c349ead889ee28176caaf492143424be42", size = 58514, upload-time = "2026-05-26T09:56:04.083Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "regex" +version = "2026.5.9" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/dc/0e/49aee608ad09480e7fd276898c99ec6192985fa331abe4eb3a986094490b/regex-2026.5.9.tar.gz", hash = "sha256:a8234aa23ec39894bfe4a3f1b85616a7032481964a13ac6fc9f10de4f6fca270", size = 416074, upload-time = "2026-05-09T23:15:19.37Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c2/dc/c1f2df4027e82fc54b5a473e4b250f5139faca49a0fbe29a48668d228f34/regex-2026.5.9-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ccf5249114cc3e772ecdd88a98a86eca0fd74c61ce32a94743758c083fc05d48", size = 489445, upload-time = "2026-05-09T23:12:06.111Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/03/d2/59f01110660081cce9c0bc30ebd0b5ee250dacf658e3248ed92f01e0e8ee/regex-2026.5.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:46f1326ca6e65b0879d23ca302c0f2415aad42ff0309b9c818e7949fe19a41d8", size = 291271, upload-time = "2026-05-09T23:12:07.731Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/58/b6/14b2c84ff90ddb370c81d27503f4a0fcf071496416f4855f6cc8c5d81c35/regex-2026.5.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ef31cbfe458e21c6122ba8150ff060e0c7789ed0d26eb423f25472584920b555", size = 289212, upload-time = "2026-05-09T23:12:09.266Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/03/d0/4db86529117320de0c84afd90e70bb47434625875e34fcef9d8c127c5b16/regex-2026.5.9-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:992604d02e6d9c6d786c24a706a71ecffe1020fc1ef264044474cd81fa2c3919", size = 792310, upload-time = "2026-05-09T23:12:11.416Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/07/78/fe4800cd322f862ecffd2d553409b20d80650e5ed71b9d178f853d020b82/regex-2026.5.9-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9411dd64ca95477225734a93dfc8583b51916b8d5942f99d6cac21e09965451", size = 861721, upload-time = "2026-05-09T23:12:13.681Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b5/d0/b3618a895dd8feb897c61bb2954edd265e1767d82a01d53065d5871127a3/regex-2026.5.9-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3dd4a3ff360dfb836fecdb93a4598f9d6e2ac81e3e397125145c6221bf58cf4c", size = 906460, upload-time = "2026-05-09T23:12:15.443Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/33/6f/1481597e859ef19508b345eec4afd1416ed6e6b459c75a64026ef193aecf/regex-2026.5.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2a661a7d270a61f7cf460caee8b9fa2d5ef9e5c681234bcb9e0fe14f488e7dfc", size = 799843, upload-time = "2026-05-09T23:12:16.892Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/73/59/955734c803f59108deccba3597ae440c76b62a652733c0006e6243758420/regex-2026.5.9-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f079e50a0d3cc3cd5091fa9ff45869a2e6b2cd35895731edafb0327901a8d86d", size = 773610, upload-time = "2026-05-09T23:12:19.127Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/68/8f/70c04a236d651c81881dac42ef8538bddda6121434509d0a22d9e601503b/regex-2026.5.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4ebe8f0b5ec5a5024dc4a4c59f444c4e9afc5f2abdbb8962065b75d27fb971f9", size = 781645, upload-time = "2026-05-09T23:12:20.806Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/1d/96/05c7434d88185e5d27fe54aeb74df86bd77cd79f52f0b4eae54faa8fea70/regex-2026.5.9-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:97cf3bc1b7d7d2306772ec07366c80d9df00ff79e79cea32898883a646d2fae2", size = 854473, upload-time = "2026-05-09T23:12:22.465Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/4e/c1/6e3d8202d981f3117004bf341ee74893ba4ba8a9fbaf4b94615846550a08/regex-2026.5.9-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:0f9eede6a5cbdc02d4978090186390936e1776a7d1359b21e41014c609880bcf", size = 763311, upload-time = "2026-05-09T23:12:24.351Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/93/c7/e7737f1526b3fb32bd4c337fd6c71c3ebb5c8296fc34d11197e0955d2e35/regex-2026.5.9-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:01f0f5f55f4b64dacec85dc116d3c05fd23ad3ff037bbc73a2085775953c2611", size = 844593, upload-time = "2026-05-09T23:12:26.341Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/a5/27/0daffb1a535bb39f422c3d200f4ab023c71110ad66a32b366bee708baba0/regex-2026.5.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1268eddd8486dc561d08eee1156e40aa3a8fe10f4bdec8fa653b455fcbffd12c", size = 789167, upload-time = "2026-05-09T23:12:27.975Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ce/fc/294fe4fac4f2ed67207b17471815870c1c45b3a489e08e0ac96daea16ef6/regex-2026.5.9-cp311-cp311-win32.whl", hash = "sha256:8676474c07469d6f33dd1085ca2cd45f65785f32518f2b20e36d9953ca07f994", size = 266249, upload-time = "2026-05-09T23:12:30.141Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d0/b0/8dce459f6245bcf8f6e9f23ac9569f1a0f15c131cc0745e82b43226204cf/regex-2026.5.9-cp311-cp311-win_amd64.whl", hash = "sha256:246de9d60aa3f8538b519834dd95cbf276ea263d6a7bd5a3666dc3fa0230505b", size = 278423, upload-time = "2026-05-09T23:12:31.676Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/db/8d/f9aeff6ad63a3ef720386f2907e6d34a35a510a6e498ebad28b0fb3f6ab6/regex-2026.5.9-cp311-cp311-win_arm64.whl", hash = "sha256:d726ca3f0d76969bf1e8e477d160d3d666bbf999f6860bd314889e5345782046", size = 270420, upload-time = "2026-05-09T23:12:33.194Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/50/9b/6550044bc44e17c84d312c031c2ec42fbdb6a4ec4e29093be3a172d08772/regex-2026.5.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:57eeeb05db7979413dec5438f2db21d7ecbba787cde7a711df1a6f6df672aa06", size = 490451, upload-time = "2026-05-09T23:12:34.72Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/1e/95/fc7ba4303b5a0f92446a12ee6778ef2c6c799233f5060042a31bf390cfe9/regex-2026.5.9-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:398c521292f4c7fb807001dcd54694d3a1fcafc179a36ad9cc56f98df85930b6", size = 292112, upload-time = "2026-05-09T23:12:36.285Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/54/4b/ee27938d1b2c443e89a9a10e00d2d19aa5ee300cd3d61140644e93bb083e/regex-2026.5.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f7a7c26137296beba7784de6eba69c6a93a63ccebc385e4962fe67e267a91225", size = 289599, upload-time = "2026-05-09T23:12:38.089Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d8/dd/ba103dc19614e25f3880800ca67ce093d6e21b325d72b8383c7bf906e9fa/regex-2026.5.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6441cc660d76107934a09c22167200839a0e89604a6297f78a974e66e931d2c0", size = 796732, upload-time = "2026-05-09T23:12:40.062Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/cf/e7/f035b4fd858b050b0080bf302968dc0f59ba34e391872d54936758e6844e/regex-2026.5.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:91328f1c23d47595ca3ef0a7557fa129c5a23404b775c770697d2f35b33e0107", size = 865440, upload-time = "2026-05-09T23:12:42.059Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/0a/51/8cd301ecc899aea28124357f729f4272f44de7806fc7ca02490bfbe253e8/regex-2026.5.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:93a7860539414dddaefba2b40f8771765ae17949d4c7182b876ce429e11a8309", size = 912329, upload-time = "2026-05-09T23:12:44.373Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/cc/1e/3fbe2fa1e8cebd62f3bb7d3321cff1640aca2e240b51d9bd624aad949260/regex-2026.5.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd2810d22146b6d838acc5ec15602cb6b47920aa4e33015df3868eedfd20bab8", size = 801239, upload-time = "2026-05-09T23:12:46.268Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/17/2f/6f6008682bf2cf98040a0d3153a8e557b6ab728d7713d045cee4ce544ab8/regex-2026.5.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:daff2bdbaf1d23e52fdff7c0b7bc2048b68f978df6a4d107ac981f94caef2e66", size = 777054, upload-time = "2026-05-09T23:12:48.051Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/19/2b/eee0d20a6842ba04df4b8847a920b57ef56853f14ef85405473e586b605a/regex-2026.5.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4eeb011098fcb77af513dcef521a3dbecbf8849b1e38940759d293b7a93f5026", size = 785098, upload-time = "2026-05-09T23:12:49.851Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/4a/98/6fc1e6410feefb92159edaed5041992bfe390e8d26c721865434acbca558/regex-2026.5.9-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:ea9c8ecfa1b73c73b626534d6626e5340d429630943672b8480724f44e84b962", size = 860095, upload-time = "2026-05-09T23:12:51.666Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/18/a3/bd855e0f2cb1a978ecf6fa6bb69632dd9c3f6ea3b81cde62fde14c9daec7/regex-2026.5.9-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:cd2846168eb9ee3c513902bc8225409cb1caab31d04728b145171fa1625d9621", size = 765762, upload-time = "2026-05-09T23:12:53.413Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/dc/66/0ae8c092e60b14c79d24f8e0b7f0aea5bfbffdcab00b5483d13404d3c3a5/regex-2026.5.9-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:39617fb0cde9c0e6306dc70e3bfc096f3da793219879f7ae7aa341a69fbdcf6d", size = 852100, upload-time = "2026-05-09T23:12:55.256Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/21/de/8dfde60fc1b21c946a893ba273403b72617edb261370cb1087099a83f088/regex-2026.5.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fd03c4f0e33280d15cae17159b899245d6b7c53d21def19b263b39655061f5ce", size = 789479, upload-time = "2026-05-09T23:12:57.573Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c3/1c/bdcc98f9a4af4fdd166c74941174619ccff4726d3ce32faa8e9a2ecd38dd/regex-2026.5.9-cp312-cp312-win32.whl", hash = "sha256:164eba9b755ea6f244b0d881196fbc1fac09714e9782c9e2732b813142033c8e", size = 266699, upload-time = "2026-05-09T23:12:59.14Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/78/87/240d36864f9e48ace85f72e79ced97ceb7f27ce87739a947dcb834b4e6bc/regex-2026.5.9-cp312-cp312-win_amd64.whl", hash = "sha256:86f40a5d6444db30a125c9c9177e6b25dad981cbc37451fd838f145e6edac92e", size = 277783, upload-time = "2026-05-09T23:13:00.789Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/4f/b5/7b30f312b0669dff5beebe5b0989dc2d1a312b1a44fab852199c387a5b96/regex-2026.5.9-cp312-cp312-win_arm64.whl", hash = "sha256:96f5f58b54a063d7ea9dca08e1cf57bfe10499c4d579ee672da284f57f5f0070", size = 270513, upload-time = "2026-05-09T23:13:02.426Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/aa/da/797e91ecec6f84135da778ddce78c20e0af5d2a15c26f87a81bc3eadb6db/regex-2026.5.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:d626b84406444b165fc0ba981604edea39f0588ff1f92baa23fe50799ea9afdb", size = 490303, upload-time = "2026-05-09T23:13:04.382Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/44/da/bf30abaaa737b58f4a4b8c4a03659e02fd92092c822e0197ed9e0daab917/regex-2026.5.9-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d7bdc0ab8f3dd7e1b4f9ab88634e13374669db86bb3c72e8292f07ae313f539f", size = 292019, upload-time = "2026-05-09T23:13:06.022Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/2d/e7/d0eaf5713828417b9e5648cf81fa9bacd4961f6ab98c380c2034f8716e35/regex-2026.5.9-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a8820737949116ffff55fe18f9fc644530063ba6ebfcb8314239416e78f1347c", size = 289468, upload-time = "2026-05-09T23:13:08.214Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d3/9b/b3fdd62b003baa1a9b593cd8c8699c9651c2e80cc21a5c715707983c42d7/regex-2026.5.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa0fbdbac82cb3e4450d0ccde7d7a35607f4cb2dd9fba4b8b69bfaf8c9fa6aed", size = 796749, upload-time = "2026-05-09T23:13:10.573Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d4/30/66ab84588765f5b4b271a9ca09ef7ce2b87caa95176ec3d2ad65d7bc4902/regex-2026.5.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:57e8915c7986aa33d25e4d3629cef711cd2863f2961b10409f0c04cb8b7d9020", size = 865445, upload-time = "2026-05-09T23:13:12.523Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/1a/89/f05169e8588aac365f35ffc7f3bc3184f095ef4cfded7cfaa3c7fd5dbd89/regex-2026.5.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:508f56a89ba9cb26e4168cbc37dbd60a28d82430a9e18ad1d25fe0883c314ca2", size = 912322, upload-time = "2026-05-09T23:13:14.281Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/30/e1/c93444052cf41581f3c884ab3fb5823daf0992f11cd4388d4275ca610558/regex-2026.5.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b6d189041f15691cfa2b6c4290448ec221244d225b3f5fe9e7771b34ffcdf6e2", size = 801269, upload-time = "2026-05-09T23:13:16.569Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/50/fe/0cf96b882f540e62e8b9956599798203d599c44cf4c77917ca27400ff69b/regex-2026.5.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e82db382b44d0111b22601c509c89f64434816c9e0eef9d1989cda8cc6ff1c04", size = 777085, upload-time = "2026-05-09T23:13:18.675Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/23/5c/d78d4924e7fc875557b9e9b768423925fdfaac5549d06da7810019a9bd26/regex-2026.5.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2acfb48634f64996b57f90f39afa692ff362162722581921fe92239a59960f3c", size = 785153, upload-time = "2026-05-09T23:13:20.525Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/bf/e0/5214774090e7b4524dcea3e3c4aa74141d43043f8beb49c1599db1c8b53a/regex-2026.5.9-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d29eebfc9525db68cad3c97eedd7f754fa265aa5cd0cf4f863b2421e1b48fc9f", size = 860164, upload-time = "2026-05-09T23:13:22.263Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/6e/e1/4a57a83350319b1271f0d7a249b8672513ed928b237a741631270de6caea/regex-2026.5.9-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:debb893095e944091c16e641a6e33c1b0f4cb61ab945ec5afbf53ce7068834d8", size = 765731, upload-time = "2026-05-09T23:13:24.277Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/12/f4/499e74a20c156fc75836ee04a72a38d1a063978f600937f9760467beb1b0/regex-2026.5.9-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:d659eee77986549c9ea45b861c7567e44d6287c3dc9a4565478853f7b9fe2ff6", size = 852062, upload-time = "2026-05-09T23:13:26.125Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/5b/92/7eebc0d0a01e78629695f342ba17e0deaff8fb45e79cc0d7b98287da6e3e/regex-2026.5.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2efa205e6d98b24d1f3ab395c11aa15cdf10935bca283d0285e0499c284fba21", size = 789577, upload-time = "2026-05-09T23:13:27.814Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/05/a4/018e71f7d2ad48c1ebe6d3ae0026f9b7cb4802fd15c7cc02fdf724355102/regex-2026.5.9-cp313-cp313-win32.whl", hash = "sha256:f3844f134e834076677dd369976e9f5068679fcb8e50102fdf6b7ac96a3ec127", size = 266691, upload-time = "2026-05-09T23:13:29.549Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/e6/1d/861a93719fb9ee7dbfc3761b3797b7a3e112a5d42c6129459d2d741be9b5/regex-2026.5.9-cp313-cp313-win_amd64.whl", hash = "sha256:3527bb4942d2c14552155406cdedd906567456821848aed1cb4933a391bf5eca", size = 277747, upload-time = "2026-05-09T23:13:31.859Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d9/c6/0a2436ae4da1ba76e51cb98943c6838a9a721faa40ebe2dce07694ae34e3/regex-2026.5.9-cp313-cp313-win_arm64.whl", hash = "sha256:56a33f191f17d8c417f99945ebdc1e691d3af9605d86ec68c7e54a57e3e17af6", size = 270500, upload-time = "2026-05-09T23:13:33.525Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/e8/e9/d21346f7b60ed58789371358ed66b09d00f832e1bd7c06e55d9da5679882/regex-2026.5.9-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:01f28d868834624c934b8d2e0aa1c8341337e37831f4a012f18a5afcba4cbaf3", size = 494172, upload-time = "2026-05-09T23:13:35.935Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c4/43/fd1177a2032037c681baecdb3422ee4e1424aec4e4f470ef47793d325274/regex-2026.5.9-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:48036f6374aaa79eb3b754ec29c61d1c6b1606749d705a13f8854fa2539671f6", size = 293952, upload-time = "2026-05-09T23:13:38.307Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f2/7d/9fbf919768368d3f8a4f6c692cf2aa61e482b2b81ec6a298ace4cbf02480/regex-2026.5.9-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:b96350aa424e79d4fd6b567b344dcbe2b2d6bfc48dfe7717587e1fa6d43da6ff", size = 292314, upload-time = "2026-05-09T23:13:40.353Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/e2/6c/e41bfeecb589716843e7c4df09ba46ff2a42961457afece19059d85caeef/regex-2026.5.9-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8f3af7a4903c5c04a11a196a5aa75cdd7dd3f8508132f9fb3259d9f5908e3b88", size = 811681, upload-time = "2026-05-09T23:13:42.543Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/87/83/a5c1c525fba0aa656e88ad0face0b1829788ef4c2fb6b26df58aa1151b84/regex-2026.5.9-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7e87577720152d2caae19fe2baaf1f8d5ca12091e9e229f03915c37d1e4b9178", size = 871135, upload-time = "2026-05-09T23:13:44.326Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/18/d4/80882e799e440dd878b0979cbebf8fa4d54624a332c83037c7a701649e3f/regex-2026.5.9-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c8b9b9d294cfea3cd19c718ade7cc93492b2c4991abd9a68d0b3477ae6d8e100", size = 917265, upload-time = "2026-05-09T23:13:47.295Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ae/ff/8db60211e2286e396aad7dc7725356c502bff0901ea05bd6cdc2e1a042b9/regex-2026.5.9-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:728d8bfd28a8845c8b6bc5dc7ce010453d206396786c0765c2740cb65f37791e", size = 816311, upload-time = "2026-05-09T23:13:49.885Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/4c/47/742ef579c61730f8d268e5cf1f9ce0e37e2ea041ad0f5644724f2378e463/regex-2026.5.9-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7e30b874d341fac767d7df5a0870540541c2c054b80cfaac116e8d367a8a7ff2", size = 785498, upload-time = "2026-05-09T23:13:52.25Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/7f/ab/cb0999802dcb0fb95b1ab005e8d4163d8afdd67efc2cb6b6630ac13f8cb1/regex-2026.5.9-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:fd190e88a895a8901325fad284a3f74ea52b1da8525b76cc811fa9b1edf0ce2b", size = 801348, upload-time = "2026-05-09T23:13:54.127Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/7d/62/8ca59a24c55bc34d166eefaf3717bd77772f329fdbf984d86581e0a3571c/regex-2026.5.9-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:8e76e8161ad00694cfce6767d5dea860c6391ac5b83e5c3a39661e696f11fc7e", size = 866493, upload-time = "2026-05-09T23:13:56.067Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/8d/3d/30f2ae62cef3278bb5bb821f467277a55fb73f01032cf85997e15e8289a8/regex-2026.5.9-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:ddda5340e6c01a293027dd46232fa79eaff1b48058ce7a98f572b6445b088041", size = 772811, upload-time = "2026-05-09T23:13:57.867Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d8/ae/7d2089bcd78ad0c0161bc684339df50032acb438a7bd3305e7ddb1193cec/regex-2026.5.9-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:205109e96b3cf5adf8f4cd62bedde9487feb282b9497a3535451e5a24cd706a0", size = 856584, upload-time = "2026-05-09T23:13:59.679Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/a9/29/92ff47f75990131ea4f24ba17819e5a9d141e10819807e09addd73409af6/regex-2026.5.9-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:dfbe4579b9f08036aa7d101d1835437a20783574ac66327e6b29b4018a138081", size = 803453, upload-time = "2026-05-09T23:14:01.978Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/04/99/eff29f1037dcab36702c9ee5d6858cf1ce2336ea8ea2987f64245b99ea5e/regex-2026.5.9-cp313-cp313t-win32.whl", hash = "sha256:ed2c9e8068b614c574d8d30e543d617cf5379b0535d46f97ef00e904745a08b5", size = 269951, upload-time = "2026-05-09T23:14:03.661Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/0e/9d/8870b8981d27b22cda77bb26a5ac7ebfa9c7d9e0dea195a834a82380e748/regex-2026.5.9-cp313-cp313t-win_amd64.whl", hash = "sha256:b46b0f094dc1d3b90356c85a0bd2c9bafc4a6a190b9d6f8ddd5a033b6e088ed4", size = 281240, upload-time = "2026-05-09T23:14:05.56Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/72/b1/3379415e8f135c13ac551353397cc4fe97b4978f3cac73c5fcbcded548b8/regex-2026.5.9-cp313-cp313t-win_arm64.whl", hash = "sha256:872acc074bd29ffc9913ecdfedf6ea77502312ca44a4aa0d3779089c6069d8de", size = 272383, upload-time = "2026-05-09T23:14:07.843Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/13/3e/9c3cd292d8808b3645a2ce517e200179b6d0e903f176300bd8b542e14de5/regex-2026.5.9-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:1bd7587a2948b4085195d5a3374eaf4a425dc3e55784c038175355ecf3bbbf8a", size = 490376, upload-time = "2026-05-09T23:14:09.64Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/60/70/d43ee8a2ca0a8b68d167f21658b85520ac0574617c7f320367c5047f7556/regex-2026.5.9-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:dea2e88e1cce4522496cce630e11e67b98b7076620bc4336c3f674bc21a375f4", size = 291964, upload-time = "2026-05-09T23:14:11.424Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/21/91/9d50b433828d8e74196904e168a43abf1e6e88b2a15d47ed742456720c37/regex-2026.5.9-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:2099f7e7ff7b6aa3192312650a56e91cc091e49d50b04e4f6f8b6e28b3b27f1c", size = 289682, upload-time = "2026-05-09T23:14:13.123Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/3e/d2/b835e3cafbb9d977736912436259ff551d60919f7d7b3d37d46659c63564/regex-2026.5.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecd353045824e4477562a2ac718c25799cdaaa41f7aa925a806a8a3e6848a5b9", size = 796996, upload-time = "2026-05-09T23:14:14.923Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/2c/a6/9f992d00019166b9de01c546dd4549bc679f2a68df11b877740b0760b7c2/regex-2026.5.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:65c8c8c37377794bd5b2f3ebe51919042bf17aec802e23c833d89782ed0c78af", size = 866089, upload-time = "2026-05-09T23:14:17.757Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/e0/08/4d32af657e049b19cb62b02e46e38fe1518797bfb2203ee93a510b21b0dc/regex-2026.5.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5b73ab8afcf66c622db143d1c6fda4e58e4d537ee4f125229ad47b1ab80f34c0", size = 911530, upload-time = "2026-05-09T23:14:20.353Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d9/27/2af43dd1dc201d1fecefda64a45f4ad0995855b92724f795a777b402ee69/regex-2026.5.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0de5cf193997384ed2ca6f1cd4f78055b255d93d82d5a8cd6ba0d11c10b167e4", size = 800643, upload-time = "2026-05-09T23:14:22.265Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/a4/dd/23a249047013b5321d4a60c4d2437462086f601b061776a525e5fba2a59f/regex-2026.5.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d641a8c9a61618047796d572a39a79b26167b0411d2c3031937b2fe2d081e2cf", size = 777223, upload-time = "2026-05-09T23:14:24.179Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/94/6a/e85ed9538cd19586d0465076a4578a12e093ce776d15f3f8ce92733a8dd6/regex-2026.5.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:24b2355ef5cc9aa5b8f07d17704face1c166fdcc2290fa7bd6e6c925655a8346", size = 785760, upload-time = "2026-05-09T23:14:26.065Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/2a/c4/f25473209438638e947c55f9156fd8f236f74169229028cc99116380868e/regex-2026.5.9-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:a24852d3c29ad9e47593593d8a247c44ccc3d0548ef12c822d6ed0810affe676", size = 860891, upload-time = "2026-05-09T23:14:28.17Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f9/f7/f4f86e3c74419c37370e91f150ae0c2ef7d34b2e0e4cdd5da046a02e4022/regex-2026.5.9-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:916714069da19329ef7de197dcbc77bb3104145c7c2c864dbfbe318f46b88b14", size = 765891, upload-time = "2026-05-09T23:14:30.06Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/26/70/704d8e13765939146b1cd0ef4e2feb71d7929727d2290f026eed10095955/regex-2026.5.9-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:fa411799ca8da32a8d38d020a88faa5b6f91657d284761352940ecf9f7c3bbdd", size = 851380, upload-time = "2026-05-09T23:14:32.123Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/26/29/1a13582a8460038edc38e49f64ceb0dd7c60f5caba77571f4bf6601965d9/regex-2026.5.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1e6da47d679b7010ef27556b6e0f99771b744936db1792a10ceac6547ae1503e", size = 789350, upload-time = "2026-05-09T23:14:34.799Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/73/56/3dcafe34fc72e271d62ad9a291801e88a1457bb251c132f15fcc2e5aad1a/regex-2026.5.9-cp314-cp314-win32.whl", hash = "sha256:98bd73080e8756255137e1bd3f3f00295bbc5aa383c0e0f973920e9134d7c4ad", size = 272130, upload-time = "2026-05-09T23:14:36.729Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d0/9c/02eebf0be95efe416c664db7fb8b6b05b7a0b06a7544f2884f2558b0526f/regex-2026.5.9-cp314-cp314-win_amd64.whl", hash = "sha256:ff8d372ac2acdc048d1c19916f27ee61bc5722728458ba6ca5052f2c72d51763", size = 280999, upload-time = "2026-05-09T23:14:39.126Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/70/5a/1dd1abee76cb7a846a0bcf42fdc87e5720c3c33c24f3e37814310a513d9f/regex-2026.5.9-cp314-cp314-win_arm64.whl", hash = "sha256:e1d93bf647916292e8edcec150c07ddf3dc50179ccaf770c04a7f9e452155372", size = 273500, upload-time = "2026-05-09T23:14:41.059Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/86/c1/c5f619b0057a7965cb78ec559c1d7a45ce8c99a35bea95483d64959a93d9/regex-2026.5.9-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:83d0ee4a57d1c87cb549e195ec300b8f0ec3a82eba66d835e4e2ed8634fe4499", size = 494269, upload-time = "2026-05-09T23:14:42.869Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/05/2c/5d01f1aee33de4bbe60c8452945bfc8477ca7c5ae4450f6bfe711036cb36/regex-2026.5.9-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:d3d7eb5c9a7f6df82ed3cfac9beb93882a5cbcb5b8b157b56cb2b3b276574ac1", size = 293954, upload-time = "2026-05-09T23:14:44.822Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/7a/fe/e8988b2ae2108c6ef71bd4aa8d87fbe257976dd0810e826cd75f701c68b6/regex-2026.5.9-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:075160bf16658e16d35233300b8453aac25de4cbea808d22348b6979668e924d", size = 292405, upload-time = "2026-05-09T23:14:47.211Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/79/34/d2b0937faa7859263f7f0a3c6b103a1296306be6952dc173d0154e9a2f49/regex-2026.5.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:45375819235558a4ff1c4971dc32881f022613abdb180128f5cb4768c1765a1c", size = 811855, upload-time = "2026-05-09T23:14:49.21Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/80/fe/daf53a47457a8486db66c66c01ceb9c2303eecee3f87197f1e77eb1a736d/regex-2026.5.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ead4b163ac30a29574510cd4b3e2e985ac5290c05fc7095557d6a5f403fc31b5", size = 871189, upload-time = "2026-05-09T23:14:51.555Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/1c/75/058fc4470cbfbf57d800aff1a0022b929a3f9fa553ee10a0cdf2070eb31f/regex-2026.5.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8c6e4218fbdfbcd4f6c19efca40930d24a621bf4b48cb76bc6640543bd28ef20", size = 917485, upload-time = "2026-05-09T23:14:53.633Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/88/e7/179cfda3a28bc843b5c6cfe7f79f23489c791ed95f151083803660878432/regex-2026.5.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6351571c8a42b505eb555c0dc47d740d0fb66977dc142919eea6f4325b7c56a0", size = 816369, upload-time = "2026-05-09T23:14:56.198Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/41/90/6f0cc422071688266d344fca8462d787cba0a2c144acb25721f9a61ec265/regex-2026.5.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:002205cafd2a9e78c6290c7d1df277bf3277b3b7a30e0b4bb0dac2e2e3f7cb2d", size = 785869, upload-time = "2026-05-09T23:14:58.602Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/02/67/a31f1760f09c27b251ef39e9beb541f462cf977381d067faa764c2c0e393/regex-2026.5.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8abd33fef90b2a9efac5557d6033ca82d1195ed3a15fea5af15ba7b463c6a63b", size = 801427, upload-time = "2026-05-09T23:15:00.642Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/e3/c4/1a80654597b6bc1e1ea0494824c31200e8a956abe290afae9b19a166a148/regex-2026.5.9-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:31037c82eccb44b7ea2e9e221d7c01429430e989a1f4b91ea5a855f6017b509a", size = 866482, upload-time = "2026-05-09T23:15:03.384Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d1/11/960724e06482c08466ff5611e242e86f80062949cdf6b4b9cc317b9dd93d/regex-2026.5.9-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:5604dfd046dc37eca90250fc3be938b076c8059fa772ac0ed6f499b0f0fb0415", size = 773022, upload-time = "2026-05-09T23:15:05.625Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/50/a8/a9979c3e7918280e93159ebcab5ef1a65116dd4f3bd6091be0eae4a126e8/regex-2026.5.9-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:0e1b1b4e496afbb24f4a62aba855ee4f88f25578927697b340702e48c9ee6bc2", size = 856642, upload-time = "2026-05-09T23:15:07.966Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/fe/d4/a9b732f2f0072c0ab12227483abb24fffcb9f73f8a2b203df0a6d0434735/regex-2026.5.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:be3372b9df6ddecff6486d37e19095a7b4973137caf5512407a89f4455361f41", size = 803552, upload-time = "2026-05-09T23:15:10.215Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d5/fe/1b3113817447a1d4155e4ac76d2e072f42c0bcba2f43fa8a0e756ea2cd91/regex-2026.5.9-cp314-cp314t-win32.whl", hash = "sha256:3ddd90103f9e5c471c49c7852ecc1fe27c7e45eb99e977aefe7caa4e779f4f58", size = 275746, upload-time = "2026-05-09T23:15:12.609Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/92/73/93d42045302636c91f2e5ef588b65b84b01428f28ec77de256b1dfdfbe5c/regex-2026.5.9-cp314-cp314t-win_amd64.whl", hash = "sha256:ca518ed29c46eecba6010b15f1b9a479314d2de409536e71b6a13aa04e3b8a77", size = 285685, upload-time = "2026-05-09T23:15:15.086Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/da/80/35b4c33c804a165a7f55289afda3ea9e3eb6d15800341a2d66455c0f1f30/regex-2026.5.9-cp314-cp314t-win_arm64.whl", hash = "sha256:5e41809d2683fcde7d5a8c87a6567ba1fb1ce0de9f31bff578de00a4b2d76daa", size = 275713, upload-time = "2026-05-09T23:15:16.98Z" }, +] + +[[package]] +name = "rich" +version = "15.0.0" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, +] + +[[package]] +name = "safetensors" +version = "0.7.0" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/29/9c/6e74567782559a63bd040a236edca26fd71bc7ba88de2ef35d75df3bca5e/safetensors-0.7.0.tar.gz", hash = "sha256:07663963b67e8bd9f0b8ad15bb9163606cd27cc5a1b96235a50d8369803b96b0", size = 200878, upload-time = "2025-11-19T15:18:43.199Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/fa/47/aef6c06649039accf914afef490268e1067ed82be62bcfa5b7e886ad15e8/safetensors-0.7.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c82f4d474cf725255d9e6acf17252991c3c8aac038d6ef363a4bf8be2f6db517", size = 467781, upload-time = "2025-11-19T15:18:35.84Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/e8/00/374c0c068e30cd31f1e1b46b4b5738168ec79e7689ca82ee93ddfea05109/safetensors-0.7.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:94fd4858284736bb67a897a41608b5b0c2496c9bdb3bf2af1fa3409127f20d57", size = 447058, upload-time = "2025-11-19T15:18:34.416Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f1/06/578ffed52c2296f93d7fd2d844cabfa92be51a587c38c8afbb8ae449ca89/safetensors-0.7.0-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e07d91d0c92a31200f25351f4acb2bc6aff7f48094e13ebb1d0fb995b54b6542", size = 491748, upload-time = "2025-11-19T15:18:09.79Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ae/33/1debbbb70e4791dde185edb9413d1fe01619255abb64b300157d7f15dddd/safetensors-0.7.0-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8469155f4cb518bafb4acf4865e8bb9d6804110d2d9bdcaa78564b9fd841e104", size = 503881, upload-time = "2025-11-19T15:18:16.145Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/8e/1c/40c2ca924d60792c3be509833df711b553c60effbd91da6f5284a83f7122/safetensors-0.7.0-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:54bef08bf00a2bff599982f6b08e8770e09cc012d7bba00783fc7ea38f1fb37d", size = 623463, upload-time = "2025-11-19T15:18:21.11Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/9b/3a/13784a9364bd43b0d61eef4bea2845039bc2030458b16594a1bd787ae26e/safetensors-0.7.0-cp38-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:42cb091236206bb2016d245c377ed383aa7f78691748f3bb6ee1bfa51ae2ce6a", size = 532855, upload-time = "2025-11-19T15:18:25.719Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/a0/60/429e9b1cb3fc651937727befe258ea24122d9663e4d5709a48c9cbfceecb/safetensors-0.7.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dac7252938f0696ddea46f5e855dd3138444e82236e3be475f54929f0c510d48", size = 507152, upload-time = "2025-11-19T15:18:33.023Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/3c/a8/4b45e4e059270d17af60359713ffd83f97900d45a6afa73aaa0d737d48b6/safetensors-0.7.0-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1d060c70284127fa805085d8f10fbd0962792aed71879d00864acda69dbab981", size = 541856, upload-time = "2025-11-19T15:18:31.075Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/06/87/d26d8407c44175d8ae164a95b5a62707fcc445f3c0c56108e37d98070a3d/safetensors-0.7.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:cdab83a366799fa730f90a4ebb563e494f28e9e92c4819e556152ad55e43591b", size = 674060, upload-time = "2025-11-19T15:18:37.211Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/11/f5/57644a2ff08dc6325816ba7217e5095f17269dada2554b658442c66aed51/safetensors-0.7.0-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:672132907fcad9f2aedcb705b2d7b3b93354a2aec1b2f706c4db852abe338f85", size = 771715, upload-time = "2025-11-19T15:18:38.689Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/86/31/17883e13a814bd278ae6e266b13282a01049b0c81341da7fd0e3e71a80a3/safetensors-0.7.0-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:5d72abdb8a4d56d4020713724ba81dac065fedb7f3667151c4a637f1d3fb26c0", size = 714377, upload-time = "2025-11-19T15:18:40.162Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/4a/d8/0c8a7dc9b41dcac53c4cbf9df2b9c83e0e0097203de8b37a712b345c0be5/safetensors-0.7.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b0f6d66c1c538d5a94a73aa9ddca8ccc4227e6c9ff555322ea40bdd142391dd4", size = 677368, upload-time = "2025-11-19T15:18:41.627Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/05/e5/cb4b713c8a93469e3c5be7c3f8d77d307e65fe89673e731f5c2bfd0a9237/safetensors-0.7.0-cp38-abi3-win32.whl", hash = "sha256:c74af94bf3ac15ac4d0f2a7c7b4663a15f8c2ab15ed0fc7531ca61d0835eccba", size = 326423, upload-time = "2025-11-19T15:18:45.74Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/5d/e6/ec8471c8072382cb91233ba7267fd931219753bb43814cbc71757bfd4dab/safetensors-0.7.0-cp38-abi3-win_amd64.whl", hash = "sha256:d1239932053f56f3456f32eb9625590cc7582e905021f94636202a864d470755", size = 341380, upload-time = "2025-11-19T15:18:44.427Z" }, +] + +[[package]] +name = "sentencepiece" +version = "0.2.1" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/15/15/2e7a025fc62d764b151ae6d0f2a92f8081755ebe8d4a64099accc6f77ba6/sentencepiece-0.2.1.tar.gz", hash = "sha256:8138cec27c2f2282f4a34d9a016e3374cd40e5c6e9cb335063db66a0a3b71fad", size = 3228515, upload-time = "2025-08-12T07:00:51.718Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d8/15/46afbab00733d81788b64be430ca1b93011bb9388527958e26cc31832de5/sentencepiece-0.2.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6356d0986b8b8dc351b943150fcd81a1c6e6e4d439772e8584c64230e58ca987", size = 1942560, upload-time = "2025-08-12T06:59:25.82Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/fa/79/7c01b8ef98a0567e9d84a4e7a910f8e7074fcbf398a5cd76f93f4b9316f9/sentencepiece-0.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f8ba89a3acb3dc1ae90f65ec1894b0b9596fdb98ab003ff38e058f898b39bc7", size = 1325385, upload-time = "2025-08-12T06:59:27.722Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/bb/88/2b41e07bd24f33dcf2f18ec3b74247aa4af3526bad8907b8727ea3caba03/sentencepiece-0.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:02593eca45440ef39247cee8c47322a34bdcc1d8ae83ad28ba5a899a2cf8d79a", size = 1253319, upload-time = "2025-08-12T06:59:29.306Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/a0/54/38a1af0c6210a3c6f95aa46d23d6640636d020fba7135cd0d9a84ada05a7/sentencepiece-0.2.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a0d15781a171d188b661ae4bde1d998c303f6bd8621498c50c671bd45a4798e", size = 1316162, upload-time = "2025-08-12T06:59:30.914Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ef/66/fb191403ade791ad2c3c1e72fe8413e63781b08cfa3aa4c9dfc536d6e795/sentencepiece-0.2.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4f5a3e0d9f445ed9d66c0fec47d4b23d12cfc858b407a03c194c1b26c2ac2a63", size = 1387785, upload-time = "2025-08-12T06:59:32.491Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/a9/2d/3bd9b08e70067b2124518b308db6a84a4f8901cc8a4317e2e4288cdd9b4d/sentencepiece-0.2.1-cp311-cp311-win32.whl", hash = "sha256:6d297a1748d429ba8534eebe5535448d78b8acc32d00a29b49acf28102eeb094", size = 999555, upload-time = "2025-08-12T06:59:34.475Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/32/b8/f709977f5fda195ae1ea24f24e7c581163b6f142b1005bc3d0bbfe4d7082/sentencepiece-0.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:82d9ead6591015f009cb1be1cb1c015d5e6f04046dbb8c9588b931e869a29728", size = 1054617, upload-time = "2025-08-12T06:59:36.461Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/7a/40/a1fc23be23067da0f703709797b464e8a30a1c78cc8a687120cd58d4d509/sentencepiece-0.2.1-cp311-cp311-win_arm64.whl", hash = "sha256:39f8651bd10974eafb9834ce30d9bcf5b73e1fc798a7f7d2528f9820ca86e119", size = 1033877, upload-time = "2025-08-12T06:59:38.391Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/4a/be/32ce495aa1d0e0c323dcb1ba87096037358edee539cac5baf8755a6bd396/sentencepiece-0.2.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:57cae326c8727de58c85977b175af132a7138d84c764635d7e71bbee7e774133", size = 1943152, upload-time = "2025-08-12T06:59:40.048Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/88/7e/ff23008899a58678e98c6ff592bf4d368eee5a71af96d0df6b38a039dd4f/sentencepiece-0.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:56dd39a3c4d6493db3cdca7e8cc68c6b633f0d4195495cbadfcf5af8a22d05a6", size = 1325651, upload-time = "2025-08-12T06:59:41.536Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/19/84/42eb3ce4796777a1b5d3699dfd4dca85113e68b637f194a6c8d786f16a04/sentencepiece-0.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d9381351182ff9888cc80e41c632e7e274b106f450de33d67a9e8f6043da6f76", size = 1253645, upload-time = "2025-08-12T06:59:42.903Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/89/fa/d3d5ebcba3cb9e6d3775a096251860c41a6bc53a1b9461151df83fe93255/sentencepiece-0.2.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:99f955df238021bf11f0fc37cdb54fd5e5b5f7fd30ecc3d93fb48b6815437167", size = 1316273, upload-time = "2025-08-12T06:59:44.476Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/04/88/14f2f4a2b922d8b39be45bf63d79e6cd3a9b2f248b2fcb98a69b12af12f5/sentencepiece-0.2.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0cdfecef430d985f1c2bcbfff3defd1d95dae876fbd0173376012d2d7d24044b", size = 1387881, upload-time = "2025-08-12T06:59:46.09Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/fd/b8/903e5ccb77b4ef140605d5d71b4f9e0ad95d456d6184688073ed11712809/sentencepiece-0.2.1-cp312-cp312-win32.whl", hash = "sha256:a483fd29a34c3e34c39ac5556b0a90942bec253d260235729e50976f5dba1068", size = 999540, upload-time = "2025-08-12T06:59:48.023Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/2d/81/92df5673c067148c2545b1bfe49adfd775bcc3a169a047f5a0e6575ddaca/sentencepiece-0.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:4cdc7c36234fda305e85c32949c5211faaf8dd886096c7cea289ddc12a2d02de", size = 1054671, upload-time = "2025-08-12T06:59:49.895Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/fe/02/c5e3bc518655d714622bec87d83db9cdba1cd0619a4a04e2109751c4f47f/sentencepiece-0.2.1-cp312-cp312-win_arm64.whl", hash = "sha256:daeb5e9e9fcad012324807856113708614d534f596d5008638eb9b40112cd9e4", size = 1033923, upload-time = "2025-08-12T06:59:51.952Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ba/4a/85fbe1706d4d04a7e826b53f327c4b80f849cf1c7b7c5e31a20a97d8f28b/sentencepiece-0.2.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:dcd8161eee7b41aae57ded06272905dbd680a0a04b91edd0f64790c796b2f706", size = 1943150, upload-time = "2025-08-12T06:59:53.588Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c2/83/4cfb393e287509fc2155480b9d184706ef8d9fa8cbf5505d02a5792bf220/sentencepiece-0.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c6c8f42949f419ff8c7e9960dbadcfbc982d7b5efc2f6748210d3dd53a7de062", size = 1325651, upload-time = "2025-08-12T06:59:55.073Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/8d/de/5a007fb53b1ab0aafc69d11a5a3dd72a289d5a3e78dcf2c3a3d9b14ffe93/sentencepiece-0.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:097f3394e99456e9e4efba1737c3749d7e23563dd1588ce71a3d007f25475fff", size = 1253641, upload-time = "2025-08-12T06:59:56.562Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/2c/d2/f552be5928105588f4f4d66ee37dd4c61460d8097e62d0e2e0eec41bc61d/sentencepiece-0.2.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d7b670879c370d350557edabadbad1f6561a9e6968126e6debca4029e5547820", size = 1316271, upload-time = "2025-08-12T06:59:58.109Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/96/df/0cfe748ace5485be740fed9476dee7877f109da32ed0d280312c94ec259f/sentencepiece-0.2.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c7f0fd2f2693309e6628aeeb2e2faf6edd221134dfccac3308ca0de01f8dab47", size = 1387882, upload-time = "2025-08-12T07:00:00.701Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ac/dd/f7774d42a881ced8e1739f393ab1e82ece39fc9abd4779e28050c2e975b5/sentencepiece-0.2.1-cp313-cp313-win32.whl", hash = "sha256:92b3816aa2339355fda2c8c4e021a5de92180b00aaccaf5e2808972e77a4b22f", size = 999541, upload-time = "2025-08-12T07:00:02.709Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/dd/e9/932b9eae6fd7019548321eee1ab8d5e3b3d1294df9d9a0c9ac517c7b636d/sentencepiece-0.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:10ed3dab2044c47f7a2e7b4969b0c430420cdd45735d78c8f853191fa0e3148b", size = 1054669, upload-time = "2025-08-12T07:00:04.915Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c9/3a/76488a00ea7d6931689cda28726a1447d66bf1a4837943489314593d5596/sentencepiece-0.2.1-cp313-cp313-win_arm64.whl", hash = "sha256:ac650534e2251083c5f75dde4ff28896ce7c8904133dc8fef42780f4d5588fcd", size = 1033922, upload-time = "2025-08-12T07:00:06.496Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/4a/b6/08fe2ce819e02ccb0296f4843e3f195764ce9829cbda61b7513f29b95718/sentencepiece-0.2.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:8dd4b477a7b069648d19363aad0cab9bad2f4e83b2d179be668efa672500dc94", size = 1946052, upload-time = "2025-08-12T07:00:08.136Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ab/d9/1ea0e740591ff4c6fc2b6eb1d7510d02f3fb885093f19b2f3abd1363b402/sentencepiece-0.2.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0c0f672da370cc490e4c59d89e12289778310a0e71d176c541e4834759e1ae07", size = 1327408, upload-time = "2025-08-12T07:00:09.572Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/99/7e/1fb26e8a21613f6200e1ab88824d5d203714162cf2883248b517deb500b7/sentencepiece-0.2.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:ad8493bea8432dae8d6830365352350f3b4144415a1d09c4c8cb8d30cf3b6c3c", size = 1254857, upload-time = "2025-08-12T07:00:11.021Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/bc/85/c72fd1f3c7a6010544d6ae07f8ddb38b5e2a7e33bd4318f87266c0bbafbf/sentencepiece-0.2.1-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b81a24733726e3678d2db63619acc5a8dccd074f7aa7a54ecd5ca33ca6d2d596", size = 1315722, upload-time = "2025-08-12T07:00:12.989Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/4a/e8/661e5bd82a8aa641fd6c1020bd0e890ef73230a2b7215ddf9c8cd8e941c2/sentencepiece-0.2.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0a81799d0a68d618e89063fb423c3001a034c893069135ffe51fee439ae474d6", size = 1387452, upload-time = "2025-08-12T07:00:15.088Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/99/5e/ae66c361023a470afcbc1fbb8da722c72ea678a2fcd9a18f1a12598c7501/sentencepiece-0.2.1-cp313-cp313t-win32.whl", hash = "sha256:89a3ea015517c42c0341d0d962f3e6aaf2cf10d71b1932d475c44ba48d00aa2b", size = 1002501, upload-time = "2025-08-12T07:00:16.966Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c1/03/d332828c4ff764e16c1b56c2c8f9a33488bbe796b53fb6b9c4205ddbf167/sentencepiece-0.2.1-cp313-cp313t-win_amd64.whl", hash = "sha256:33f068c9382dc2e7c228eedfd8163b52baa86bb92f50d0488bf2b7da7032e484", size = 1057555, upload-time = "2025-08-12T07:00:18.573Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/88/14/5aee0bf0864df9bd82bd59e7711362908e4935e3f9cdc1f57246b5d5c9b9/sentencepiece-0.2.1-cp313-cp313t-win_arm64.whl", hash = "sha256:b3616ad246f360e52c85781e47682d31abfb6554c779e42b65333d4b5f44ecc0", size = 1036042, upload-time = "2025-08-12T07:00:20.209Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/24/9c/89eb8b2052f720a612478baf11c8227dcf1dc28cd4ea4c0c19506b5af2a2/sentencepiece-0.2.1-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:5d0350b686c320068702116276cfb26c066dc7e65cfef173980b11bb4d606719", size = 1943147, upload-time = "2025-08-12T07:00:21.809Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/82/0b/a1432bc87f97c2ace36386ca23e8bd3b91fb40581b5e6148d24b24186419/sentencepiece-0.2.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:c7f54a31cde6fa5cb030370566f68152a742f433f8d2be458463d06c208aef33", size = 1325624, upload-time = "2025-08-12T07:00:23.289Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ea/99/bbe054ebb5a5039457c590e0a4156ed073fb0fe9ce4f7523404dd5b37463/sentencepiece-0.2.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c83b85ab2d6576607f31df77ff86f28182be4a8de6d175d2c33ca609925f5da1", size = 1253670, upload-time = "2025-08-12T07:00:24.69Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/19/ad/d5c7075f701bd97971d7c2ac2904f227566f51ef0838dfbdfdccb58cd212/sentencepiece-0.2.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1855f57db07b51fb51ed6c9c452f570624d2b169b36f0f79ef71a6e6c618cd8b", size = 1316247, upload-time = "2025-08-12T07:00:26.435Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/fb/03/35fbe5f3d9a7435eebd0b473e09584bd3cc354ce118b960445b060d33781/sentencepiece-0.2.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01e6912125cb45d3792f530a4d38f8e21bf884d6b4d4ade1b2de5cf7a8d2a52b", size = 1387894, upload-time = "2025-08-12T07:00:28.339Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/dc/aa/956ef729aafb6c8f9c443104c9636489093bb5c61d6b90fc27aa1a865574/sentencepiece-0.2.1-cp314-cp314-win32.whl", hash = "sha256:c415c9de1447e0a74ae3fdb2e52f967cb544113a3a5ce3a194df185cbc1f962f", size = 1096698, upload-time = "2025-08-12T07:00:29.764Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b8/cb/fe400d8836952cc535c81a0ce47dc6875160e5fedb71d2d9ff0e9894c2a6/sentencepiece-0.2.1-cp314-cp314-win_amd64.whl", hash = "sha256:881b2e44b14fc19feade3cbed314be37de639fc415375cefaa5bc81a4be137fd", size = 1155115, upload-time = "2025-08-12T07:00:32.865Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/32/89/047921cf70f36c7b6b6390876b2399b3633ab73b8d0cb857e5a964238941/sentencepiece-0.2.1-cp314-cp314-win_arm64.whl", hash = "sha256:2005242a16d2dc3ac5fe18aa7667549134d37854823df4c4db244752453b78a8", size = 1133890, upload-time = "2025-08-12T07:00:34.763Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/a1/11/5b414b9fae6255b5fb1e22e2ed3dc3a72d3a694e5703910e640ac78346bb/sentencepiece-0.2.1-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:a19adcec27c524cb7069a1c741060add95f942d1cbf7ad0d104dffa0a7d28a2b", size = 1946081, upload-time = "2025-08-12T07:00:36.97Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/77/eb/7a5682bb25824db8545f8e5662e7f3e32d72a508fdce086029d89695106b/sentencepiece-0.2.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:e37e4b4c4a11662b5db521def4e44d4d30ae69a1743241412a93ae40fdcab4bb", size = 1327406, upload-time = "2025-08-12T07:00:38.669Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/03/b0/811dae8fb9f2784e138785d481469788f2e0d0c109c5737372454415f55f/sentencepiece-0.2.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:477c81505db072b3ab627e7eab972ea1025331bd3a92bacbf798df2b75ea86ec", size = 1254846, upload-time = "2025-08-12T07:00:40.611Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ef/23/195b2e7ec85ebb6a547969f60b723c7aca5a75800ece6cc3f41da872d14e/sentencepiece-0.2.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:010f025a544ef770bb395091d57cb94deb9652d8972e0d09f71d85d5a0816c8c", size = 1315721, upload-time = "2025-08-12T07:00:42.914Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/7e/aa/553dbe4178b5f23eb28e59393dddd64186178b56b81d9b8d5c3ff1c28395/sentencepiece-0.2.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:733e59ff1794d26db706cd41fc2d7ca5f6c64a820709cb801dc0ea31780d64ab", size = 1387458, upload-time = "2025-08-12T07:00:44.56Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/66/7c/08ff0012507297a4dd74a5420fdc0eb9e3e80f4e88cab1538d7f28db303d/sentencepiece-0.2.1-cp314-cp314t-win32.whl", hash = "sha256:d3233770f78e637dc8b1fda2cd7c3b99ec77e7505041934188a4e7fe751de3b0", size = 1099765, upload-time = "2025-08-12T07:00:46.058Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/91/d5/2a69e1ce15881beb9ddfc7e3f998322f5cedcd5e4d244cb74dade9441663/sentencepiece-0.2.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e4366c97b68218fd30ea72d70c525e6e78a6c0a88650f57ac4c43c63b234a9d", size = 1157807, upload-time = "2025-08-12T07:00:47.673Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f3/16/54f611fcfc2d1c46cbe3ec4169780b2cfa7cf63708ef2b71611136db7513/sentencepiece-0.2.1-cp314-cp314t-win_arm64.whl", hash = "sha256:105e36e75cbac1292642045458e8da677b2342dcd33df503e640f0b457cb6751", size = 1136264, upload-time = "2025-08-12T07:00:49.485Z" }, +] + +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, +] + +[[package]] +name = "sparkle-engine" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "fastapi" }, + { name = "mlx" }, + { name = "mlx-lm" }, + { name = "numpy" }, + { name = "textual" }, + { name = "uvicorn" }, +] + +[package.dev-dependencies] +dev = [ + { name = "httpx" }, + { name = "nanobind" }, + { name = "pytest" }, + { name = "pytest-asyncio" }, +] + +[package.metadata] +requires-dist = [ + { name = "fastapi", specifier = ">=0.115" }, + { name = "mlx", specifier = ">=0.31" }, + { name = "mlx-lm", specifier = ">=0.31" }, + { name = "numpy", specifier = ">=2.0" }, + { name = "textual", specifier = ">=0.80" }, + { name = "uvicorn", specifier = ">=0.30" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "httpx", specifier = ">=0.27" }, + { name = "nanobind", specifier = ">=2.12.0" }, + { name = "pytest", specifier = ">=8" }, + { name = "pytest-asyncio", specifier = ">=0.23" }, +] + +[[package]] +name = "starlette" +version = "1.3.1" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "anyio" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/eb/e3/7c1dc7381d9f8ab7d854328ebfa884e62cb3f3d8549ddfd37c7814f42afa/starlette-1.3.1.tar.gz", hash = "sha256:05d0213193f2fbaae60e2ecb593b4add4262ad4e46536b54abe36f11a71724e0", size = 2703240, upload-time = "2026-06-12T09:23:11.602Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ec/bb/2799cc2ede3ed41131f8975621e7213dfc7ef4acbbaadfa440f32500c370/starlette-1.3.1-py3-none-any.whl", hash = "sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6", size = 73632, upload-time = "2026-06-12T09:23:10.017Z" }, +] + +[[package]] +name = "textual" +version = "8.2.8" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "markdown-it-py", extra = ["linkify"] }, + { name = "mdit-py-plugins" }, + { name = "platformdirs" }, + { name = "pygments" }, + { name = "rich" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/00/21/39a76b01bd5eea82a04baaca7580e105d8c59450df03998345bb2cfb307b/textual-8.2.8.tar.gz", hash = "sha256:3f106a9fbc73e39dd266c9712432087de78a6d644084c7c241d6a25c3169115b", size = 1860502, upload-time = "2026-06-30T06:51:24.495Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/fb/be/35261223d9416a0751cdff1c7b4a6f881387218a12d439fe22fefebc8c04/textual-8.2.8-py3-none-any.whl", hash = "sha256:267375fd402dc8d981457212efa71f0e3365fd17bba144ba9bb3ed7563cb374a", size = 731418, upload-time = "2026-06-30T06:51:26.364Z" }, +] + +[[package]] +name = "tokenizers" +version = "0.22.2" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "huggingface-hub" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/73/6f/f80cfef4a312e1fb34baf7d85c72d4411afde10978d4657f8cdd811d3ccc/tokenizers-0.22.2.tar.gz", hash = "sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917", size = 372115, upload-time = "2026-01-05T10:45:15.988Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/92/97/5dbfabf04c7e348e655e907ed27913e03db0923abb5dfdd120d7b25630e1/tokenizers-0.22.2-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c", size = 3100275, upload-time = "2026-01-05T10:41:02.158Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/2e/47/174dca0502ef88b28f1c9e06b73ce33500eedfac7a7692108aec220464e7/tokenizers-0.22.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001", size = 2981472, upload-time = "2026-01-05T10:41:00.276Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d6/84/7990e799f1309a8b87af6b948f31edaa12a3ed22d11b352eaf4f4b2e5753/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7", size = 3290736, upload-time = "2026-01-05T10:40:32.165Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/78/59/09d0d9ba94dcd5f4f1368d4858d24546b4bdc0231c2354aa31d6199f0399/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd", size = 3168835, upload-time = "2026-01-05T10:40:38.847Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/47/50/b3ebb4243e7160bda8d34b731e54dd8ab8b133e50775872e7a434e524c28/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5", size = 3521673, upload-time = "2026-01-05T10:40:56.614Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/e0/fa/89f4cb9e08df770b57adb96f8cbb7e22695a4cb6c2bd5f0c4f0ebcf33b66/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e", size = 3724818, upload-time = "2026-01-05T10:40:44.507Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/64/04/ca2363f0bfbe3b3d36e95bf67e56a4c88c8e3362b658e616d1ac185d47f2/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b", size = 3379195, upload-time = "2026-01-05T10:40:51.139Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/2e/76/932be4b50ef6ccedf9d3c6639b056a967a86258c6d9200643f01269211ca/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67", size = 3274982, upload-time = "2026-01-05T10:40:58.331Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/1d/28/5f9f5a4cc211b69e89420980e483831bcc29dade307955cc9dc858a40f01/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4", size = 9478245, upload-time = "2026-01-05T10:41:04.053Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/6c/fb/66e2da4704d6aadebf8cb39f1d6d1957df667ab24cff2326b77cda0dcb85/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a", size = 9560069, upload-time = "2026-01-05T10:45:10.673Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/16/04/fed398b05caa87ce9b1a1bb5166645e38196081b225059a6edaff6440fac/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a", size = 9899263, upload-time = "2026-01-05T10:45:12.559Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/05/a1/d62dfe7376beaaf1394917e0f8e93ee5f67fea8fcf4107501db35996586b/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5", size = 10033429, upload-time = "2026-01-05T10:45:14.333Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/fd/18/a545c4ea42af3df6effd7d13d250ba77a0a86fb20393143bbb9a92e434d4/tokenizers-0.22.2-cp39-abi3-win32.whl", hash = "sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92", size = 2502363, upload-time = "2026-01-05T10:45:20.593Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/65/71/0670843133a43d43070abeb1949abfdef12a86d490bea9cd9e18e37c5ff7/tokenizers-0.22.2-cp39-abi3-win_amd64.whl", hash = "sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48", size = 2747786, upload-time = "2026-01-05T10:45:18.411Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/72/f4/0de46cfa12cdcbcd464cc59fde36912af405696f687e53a091fb432f694c/tokenizers-0.22.2-cp39-abi3-win_arm64.whl", hash = "sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc", size = 2612133, upload-time = "2026-01-05T10:45:17.232Z" }, +] + +[[package]] +name = "tqdm" +version = "4.68.1" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/06/b3/36c8ecf72e8925200671613332db156d84b99b3aee742a41c1938ebb0808/tqdm-4.68.1.tar.gz", hash = "sha256:fc163d96b287bd031e1aa24421ce4411b25559bd0a1be4fe649bdaa4d2c02bf5", size = 171236, upload-time = "2026-06-05T17:23:15.267Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/47/aa/218a0eb34de1f753c83e4d0d1c8e7c4cef27f20dcb8342e024f63a80dc86/tqdm-4.68.1-py3-none-any.whl", hash = "sha256:fea4a90e4023f764914569f7802a297277c5ab1a66be5144143e142e1a4031d8", size = 78354, upload-time = "2026-06-05T17:23:13.654Z" }, +] + +[[package]] +name = "transformers" +version = "5.10.2" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "huggingface-hub" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "regex" }, + { name = "safetensors" }, + { name = "tokenizers" }, + { name = "tqdm" }, + { name = "typer" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/8d/38/d5f978bd5091019e89aef29b9a831f5cd70f2598963a3ead8b9570cab592/transformers-5.10.2.tar.gz", hash = "sha256:f9a44b9c8ca9ab1156b467f574d832ea066284299c2fd0ed84641ccb592751fc", size = 8799687, upload-time = "2026-06-04T18:43:49.119Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/73/6f/e1564b0cc182afa05e219a8e09a8e770ffaab879b6b824b56c819bd221da/transformers-5.10.2-py3-none-any.whl", hash = "sha256:8a669db546f82c7c3618cb46ceb0f0afd89292bc70f319c058f8332ec63e268d", size = 11003830, upload-time = "2026-06-04T18:43:45.303Z" }, +] + +[[package]] +name = "typer" +version = "0.25.1" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "click" }, + { name = "rich" }, + { name = "shellingham" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/e4/51/9aed62104cea109b820bbd6c14245af756112017d309da813ef107d42e7e/typer-0.25.1.tar.gz", hash = "sha256:9616eb8853a09ffeabab1698952f33c6f29ffdbceb4eaeecf571880e8d7664cc", size = 122276, upload-time = "2026-04-30T19:32:16.964Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/3f/f9/2b3ff4e56e5fa7debfaf9eb135d0da96f3e9a1d5b27222223c7296336e5f/typer-0.25.1-py3-none-any.whl", hash = "sha256:75caa44ed46a03fb2dab8808753ffacdbfea88495e74c85a28c5eefcf5f39c89", size = 58409, upload-time = "2026-04-30T19:32:18.271Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] + +[[package]] +name = "uc-micro-py" +version = "2.0.0" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/78/67/9a363818028526e2d4579334460df777115bdec1bb77c08f9db88f6389f2/uc_micro_py-2.0.0.tar.gz", hash = "sha256:c53691e495c8db60e16ffc4861a35469b0ba0821fe409a8a7a0a71864d33a811", size = 6611, upload-time = "2026-03-01T06:31:27.526Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/61/73/d21edf5b204d1467e06500080a50f79d49ef2b997c79123a536d4a17d97c/uc_micro_py-2.0.0-py3-none-any.whl", hash = "sha256:3603a3859af53e5a39bc7677713c78ea6589ff188d70f4fee165db88e22b242c", size = 6383, upload-time = "2026-03-01T06:31:26.257Z" }, +] + +[[package]] +name = "uvicorn" +version = "0.51.0" +source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/a2/65/b7c6c443ccc58678c91e1e973bbe2a878591538655d6e1d47f24ba1c51f3/uvicorn-0.51.0.tar.gz", hash = "sha256:f6f4b69b657c312f516dd2d268ab9ae6f254b11e4bac504f37b2ab58b24dd0b0", size = 94412, upload-time = "2026-07-08T10:59:05.962Z" } +wheels = [ + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/45/ec/dbb7e5a6b91f86bfb9eb7d2988a2730907b6a729875b949c7f022e8b88fa/uvicorn-0.51.0-py3-none-any.whl", hash = "sha256:5d38af6cd620f2ae3849fb44fd4879e0890aa1febe8d47eb355fb45d93fe6a5b", size = 73219, upload-time = "2026-07-08T10:59:04.44Z" }, +]

G63cSlM=d>GnKTi!p1vtmE z!l9A^=GnsNfJba=xS}HJIpI$!8Fp{v;u@i*RH$A=o`exWkD|vbSa_723ok4g7b5>f z?`@JTs$hqpgJG5vXB$EMrwogt{N(AuVjT(ew4w3raif$}d2%>WN`8zX1k!1JoiX*O z-IE*RJGX(N)VL!XSckpt;1>4t6iP9&-GF{6?sJdSr~CZ*Pta2{mCS8L3U^}FMKSy_ zVZ9D>*_Czg5xh&Wkw=7cDc)haYMG4my{**6XP`gcc8X&EPM*#>Y-r~<^b}606yw^( zlPSf~pjHIymlfQS67;BxEsE=pu-J8Y%_A>&_n}*gPi!BmrMN&Yc7`e%@X+A*BKO^!v5DqLi=?yIY4b?c4sI;Q5teeS2_jRq#r-TJv62N85oNtVuJEwu1`! z+D`0YPtTzW{`6SzFUNz@;WJU;%t*@+1+rR}5@~Ibg>f8~NDBgby2X@7!)MOmwmPkIPP~?s&cECsWPcEw}BCZ@e&v-|rJ&OSySETYp}sUQVR#^=X&l=J_+8!n*8x zmw&j~br+?o$A$MgtZ;W$zE?0W#fTmi)}{E!h`B>y_&aZIa}>^=HUX+zW1we>b`8ODE{yGiLJwl_GoKQ>(okdxZOUjqTG-xJGj#< zFFq#Hv;SLpi%TlOot>4qnR~6Gqbg$ zX4a!oa78|RQEN|31Mwj~%yi?cskPZI{hrq|eiFxi@AC2Q0v^r*cat`74h8t-fgYj| zd1dENQlcMDVxuQq!id!IIn^-c%m$^w956CX4!L*rLEY3 zg!wA?iBXoth+Tel#vEJQzwe{l@a(Qbj(V?o4+rv{QR8LR{QChM%gIVKs{%MV&_i&R3=}z1VUOZH5=TJE z0!K6iweTmm)?4;({9$z*uf&XBSf7!UnEkcoph7IOKD!5mxK~^F>MSRyw}8}6Z!}$2 zh|)#p`@-d)2+=v?%N5AXKe2<+?`bFk*A(3ZI zjUnC@tcmyD&|eJZ!3-*(w;a#JqcNx$ogTQEHE$J+;QXxi`Q@oo1C+-w^&)ElAva`& z^xlxhd5=_(Tsq{vWwIP5!tU+VaxS6Wv0|r%wzCFnu`I*WFv~_`^@zquCk!Sx>$4rV z#FSsy9|}p!I;pnHlvNWcQ`U!fhj?~zsTS;)O9ggs5Ugi!zcsSq^Z$yxheVTljM%SGKU^ zRa`lgo>aV=6#RVRbAQ$k!B-H-8arbA@>ZwoDoBU&??@`+5P;iSSWY=;;gD$%9KV>) z`wy>RxTZr$(g7`5Q+8OaEv>Nq#vVtJ!}f>BaIP)4~J1u3EqYlj~VQ4oKke$mjr z7~Io)b_-z}?|ToE0j=xJ7rVN$QwK1|@s3o`;%J78Wq+xD25=93a%yQ{}E44hj$}i404ZcgBI{KFKEnJI%1Ax1q-h~qA3wx7Kp_Rs!gsu zgv9%vn8`NEjYrM=LDnGR%-$knP+iVMA^418?h%GvP457!FkYWA*r7eh7ZWsW%WFiV z#{T%qpIgV#3dL+U{&L|jv!;vImhoEGF?NzQnzo|td&OcK!DQ*r$w&a$m*J+s+Lw!w z(~_wZ-kcICyxacb);UW49AysFXZ>K_J--+6J2ETeka2(3#YfbVBaK;Y%2uH1z<^z^ zjmlfV{pFN~Q&?YXF^hY^ozCgi5%Fqwe(S=l`QXnzaJJ=#sdqDMgd zVaP=eumLmRU{eNb-u9Nq%ZHio1(WOPxq&J2yJ3IX$GN6(3n}U^p!Bspr?psuC1Mtu zBtk5F-q1exziZ&ur|aR?+ca*WN+7C9zmd3gvK`zyKv0sbMTSM`T2#Q##)+L^vRHbV z+}b@FjQWcO4GNb2I20T!=mTgsW^dT&nO zefr|m^Uge{e&b!;2(%IUyVU7N(;GU*SfeK_d61tRMSA~KBsB}wQmXhkd!gr@Hl@-o z#vg|Yyf0MuQzugB!AM8J&8^fh@c?P3;sLB@C_in{pU^6bKK{yaLdA3lQ4U;I^&tdXXf2+Jf?K&s(b47D7;YZ)OF-V z1{bAgaz++?9vN1oTD0Ex-UqYpJK8i1is!iPQ&@vT56npP$Pzd!rSe*BY9 zTHpNSlTUsDe(9fIj?caipZ)MB-}n8WeDZzx_mfZlb^LxwUt-u__~chz{@efD_kHrq zmUFA zzx7i;_5S{<+`&Gk@(Le(%rx*?<0D`-!uE>9_rv5C868 z{r$hb@x$&<{+s{H5B$jA`BVStcYf}l7~TAzKk%#mqqfAsE`{J^*W=4aO@AAh#@5B}%>ws!WH{`7BX zAN_^V-})6l@e_ai&yTFZdgO#s4S2{j)zD z{O*gN|2===+u!*8|Jgrk^?v8~|CV3#*{{3(L%;ct{15+s>D~~X3BxoSXJXs7ZQI5Z z+qP}nwr$(V#F^N(ec#31|BG+kr@NNbRdwpXX5z&-O|U8@GXn{jRU73yhH10(dJ$ZS ztZ%_zDnmlwtvYkwm2 zE=H86wIA#;*BA(fx#QJ&eezKNfaL_d?){lXhM@Dn5{Pah@VUs-eIe)Yce1u?o9EqQ*cXHyWcm+f9L0yUAC>;~1 z2G=@Hkn45dXW)1*$GIfk!n{tk#!3c8`6}k0s28`dn(9$@j77p8?X6X*wgq#fPmJLY zDMHB^LQ-%(0U;g1s0t+2cL=8nF|t_TT}6Ws56}GhE4DccpSha-5jW|9N)%lLUOdp3 z>o;k@tI^4!m)u^V8+_UVg(>d{b$0XvX24U;lp3Q29Bn(-Zd`$(NtfW)K*T*n8NW|t z9FoMMCX!7NZ4{{hW@8+~;d2IWuGaysRMdv3{Z=1?nJ^U#J$kkK>E?RAf0tG8_lO&L zNgwb&00Q4|-xf(7KV!jT@2M!=9?)A4dytV(_Yq@rR`cXa8D29o(w0@O=ZKSE^i=A+ zb9j?M`qbU=`T6Jwer~2xhWnGN`0JTX($V_bm!Tj&1aU)qv5y~e_@eE)w6(yoG}Ym= zXas%B06cFn2+)&on^)V7s$d9S?;D!0?piE5Z-KSif^NsYL;zsP5OD42< zQvW37dMLMQEM@&B)+DQ#3HQ66`-037x*q`YXbzVFjpIpPvXE~vVf`;=%%R$&ER|Bs zQ1E5am^l*4q>AyLpYI%}EUb8$e{=_)b{gi&hf~?MU$T9u5Zkr!+ybvmAsZURQfEfN zvO}4pPCXqyf<;+=67g7*?Skc`#DLYGBY(QjykZINVECe{V}Opav6$sTvuCP{6C5Z? zt%HFuGk;Q4(U_>o2*`KGX2tx_6!qaR1VzrHW73T@|Gl=fK)b#t^I!(FB^JGp+RZ_g z>&{3*0U<#L$&lUx^#Cpy<>8d-!Z-;1USh_uYQ-+HyrdZW&D1FAL(#r6mgGhhV+ttp z@{oGS7>ccLd|x`M=I<&jdd37|&G9V~5mkTSU6y!fSLJt{C;LBxA=QU1go6*iirO39 zK3CF^>*}`YpMYQbGc7~>;h?{)JC6I8saxR$cwJgSouaS*mh z60RKA2{?{v&^KT#Q@hur2J|4;!v-HBD+XB=3@(PtK#%h+d9 zWPh}yM3VXvT=iDX=b9ct-3Bvrh9@n>t}rk!mkwpkv5Z_lf#~Fl2_B0uO*;lOHr84G zYCq<)r$zPn)rGG>i^muPI0)k^EQWAK773nTyywX?Gy5V;nBLLNh^h!? zs@xXpYH&;Xb8VG2TIXggkg;T}B)DP9ahXhBlOJr3hA3N8MB$%TQ?HRU--qve8*)tO^knSNG9wOZZIP^{fc$6^>R$xK74gO4zKjTrUpZqL^?ktb*`= z-W?8hc<0_KNt9$@qx$+Rx@Qs4H=yGxgUyzNEpg-bN@YJL8B9B=2D(J!T}(r2#tfi6 zS6h>+4JJ8+cU&42mh!J0v2A__*Z{lk|M7?qPB-;7$kjw(@J-Rnz(4AXBYq@d9v@RR z?o;@F2vt39na{L42SiA@@5EWi>m6e&iGhAi;ADw;#dH`cseQ(f%zwcg1c2PpvC-VY z4kHkPWzTP|l%WurnNYJG zQ480!=FDd@@SeK5nsos4i)4+qN*#$vBn8DMhKj6g@Kp>WS8F|?T4s+VOgdyNP`+xh zcV!)!g)m_j*s$r*cD|?Hh7b4LecBV_z>*R%^=~is>q88sRt{U2S+>n=#>`ZP0deV{ z!}ord2(bl9P^@_ZTA#i#k6>WHU(OGCbD5y{ja0c4kMVKBz@+3&7QvWDCJYOWF(co6 z!>sw%`BmNxdCV za7@=TiEe8`7H)YR84L>vQKer2rVXAHBe^kd=dU1qd>_fRq+L2 zLo}YGHxQ&Bdg*Me=E9T8mgLxs%an;LKaOcZQ_78lGv(ux(UKyR@E?3wZaK*M_5Pq) zKTkqYn2DXwuoqYZn;|2s0X4hmB=T=x{k!Trso}SuGQ(3rClI^$OhtVMwg~C|IMB5R zd}&bDKuE80K-Xv0bqTf$$VI|-ee^ST#r26QPtGNp#bU~Edag__Jk*szqXws{TAOK9 z#RuV9oC9+yj6}T{f`wBwx{$edtDJXsfk@=+^)!5cmcHx8)h^I1CWR08j+q~Ko9WxU zK_f7|arI~ve-yWC8hdPCYVq;Yz^>3ggj{M|d=!2h%x%t`dzDLB5jb)43qN($gQpW{doZAB!4gWE}LPSvybQ^U3EFme0Zx($-x&)t6Mbou>5? z)^06YCyy7{!fHt)gvOS7xpyRw@I@vg69!Hw<5Ip+6WUILiMA^=!GZ)@`PURX?Q6;q zgb+k)qvY@tyWBBo_fT2gtY#!sYw~KG*CkrLH55{Y`Z*nsaH+nJtNM1^Mk+0c7mI(8 zD(Ej?nnKadJ;QxKkR_4+7E_J)n8l0fbn1f{T&Ge4D_uBr7S-f1wfZ{ARk@yHoh2uH z+l}qp8xxp*Cy0k)ocx!W08g83<)oj>Ka*$!=LzO(!-lKN~D3Rye;5} z5$F896NhI^%RZS+&i?6^Mu?kg%sWLJCh32*fly-;jQk4Ql5a=(O`+&E+0L`V2k0b+ zEdgPtv0%-e=tfiU?I(2Cz$BfC%RC}sQYqe?@SzC1>k^_ETxAGU4|_dSI>J*jA`P}NYnIAw`a zuZkFbd~H3`!t6hGeTr>4H}mTZi8dZd5r^ms_DH>1H@C~OjYvCL?&_}gD1GR}(MX4NNA4PoJvK>ldWuS|Za;`0ZQi(PfVdlOk zp!}B_*4l#4M6UTa=gPzWjR*)^RSr_e@rgq0G9fzlAa0oom0pI2ll~F1BlSc31SJo9YZ!4gpe{X5XXJ3F8lo|L?LtmOXJA(gvl5dJWL%&jaFnvnFT!vuPX zK)gdzmdGOzD1lim%8`-PV2Y~L2mL3IH>`XzE{8-t817% z0Mp8@L+_M6PN)Ks=#d7MOum;_*%R=gXmVSM-jUx+sC5(J{nuH0-B*Czi zm9(4HZO4&YI_mpTBvtI(8@ zc!$u5-tW#f{+yiBL0P5EQ{pCUOH!2u6Ml<1C^j+U&O^O#SWe=Ry=%#_zlGB@mO!h` zyv6lRz19BDjaFUXmJPJ%sCYm$<76!AZ61p9iGGT;IGPiq&okFNFBB!pZr40->Ob`L z(k3I9ZQLXFeZqZ24+jZ$f`mu1vi<^^8ExhoeC7*c%ZF?oYE>wQ`^w@$78-O_m&07i zhDn-78UAxC$WPBZfo1>hv|` z>wEp3p~k=*+n%S-I=RA%7pEL5MUOKjCWKz-)wTjJZKNf7|2mlbYDk>~p_em^(vq6p zB0G0T9|r81Bcjirqk^o5H?^Pc01nswX$t{Pw3 zWH?SuRAP!UxlsEIg7B{KC`q$HZ|wJtEXfJ4IvAsI@%{kScH)2?*RApPtPorM`Fg*w=2sW(JM;f(f zignTvM9aFW#noH!IIi5I5)_P+z>~^?YQlrHLD%6Dj>OPZK}U#z=jt=D{^kVyC2!js z`)IEZFHfWanNN8jBno&3ml0TxsZxacHJqr8ynfh3%!rJB0%y2R2lDAl1rMJ?d=Air zu+uC_SUwz-$5WJG<66=KkI|*CRG4ncP}*BBNELii(N;?eYX|m!>ZuT%*o&5m>P%ssDhQrO}SsjWfnatO|i{7C=1h zN#FQN4RHa?yPoj8IjdkRO;tjQEWo@HcrI=g;5qKL_Y-bROLs|`aD|A^W9{*-#fc1W zr0cQ_EQ%Qgb_$vr|I`M25I`1c6sk^=DfrsTk^dFvY@WA**M?+a(%rj<>>SltZoGF! zk14`44dfQ8ArYF;3pcT3^0%b0p%-uHFE&>tK^+@+J)!jU@Z_Mmc}!73*pLa_m3U#b z0hFvVp@bwT%qe%5weiCRg9ubvlHbzi`!=7&2a^CJL$V6s2sKbx4^DX!Oaal2>7;z| z*(xlXn^r*Gl74eia&pz}-U`7KQ;h5l^?dpvpaPm!Rcpt2Up#Ys=<*0|R?a$I3(qMO zCKT{{;V7r+{-)5PG~@&YSTFmny)>!zyPhfd^4e`zzS>S$rP?)jC}d@bG`pu*UgA1R z(&WJKGksTeu@aT)>KUC|T-Nhr|#Pn8yu(v}wpTnuz9EprTZ*fP#hoE!hS<8;=QO|K=#PKCQrd!0li`X+r z`7CDfDQw|?Re&5B*7EDxt%-8O#Aae9^f@<)=sqMv8okK%DP9NUg`N3b0CE}I7f?+E zM8dl`aP<&}>s{JhND3c%Ok~{6L@ozG;naFkbzFG2fBAXO7fe4!R1j*o2~uMhu~|6l z61G9iEPw)ntW(7;#+liRA%gIU4q&gfDxkW*K}J7HzYn&08wk32M>@cR%kfFm`VY&E z1{N!m4>OXV3%4OR{c)ZMsMPk#b1%&xO_ zpN-17&XY7>FyYyX@Ic1A#ug-f4O_SOz(*Y**%YGemXJfq+*p9UMUS?C$NNkOb)X00 z5sH3g+*IjQ^dbndTS|$oWwXJn7K=lvU4X%5uyJljNqn0b=kiHg)9I7^l%35j4G=ALY z<|^W@JuqZ-NVSQRS+A75ia6UE^=jvLLS5NXFq_<19g!Y-%=%-ukEUkp$e}r$bNu;c zJ+?sfWHz$K>!DB4%-_5%s#y1P&;@SNf^LcV)ohH;dQ8(79N(mKj)k{&))!Mf@A;RF zIce&3AmyC(=*D_6EG4j5K0oGfdf5I1p&aNRP$DID8J2EB3|KWB(st9bYJ5uKfCscH z2Ua&IAe3#esfpX93p;0F7~cww#2^a%+bZO@`S~(7s|V+u0FnG_H}D}{%rs+n59ODA z1L{}@8&fNfJa^H)+w}~md{{`^Z6I0-sKNHNYjga!F>bRAl~>J936lGnF&X8P(pswgfl*Ekr}qi6UC z)<9K+@ZdzFjz|G;isD#fSX90>Q6p3ZFh}zK(jG zM7IZk5y8jYj#S40MP%@!&it?U|F3-_2D;NKEkI7{US>Gmn45}0W755LRUiR%D;~^c zF9y0f^)gV1;Dbv+(KFgkt0wR^#b)N>-AFven*2Wv#g}E(%_>0$&G8s&Dw%8Ez^0JjIV53) zv|cJMz{FBohkhfm1Py9!jLvjC_FG$s+xy@pFS2d36%iXX&ECD~%qpJ>{5Kmzd{alC zkiyxMaX(%y)-BAGQ;;u?q88MDAZFKoX5I*&6Zte3F%?8Urkx@O6+^KWY2*N!^3)A{ zWg}>Wmvz_N*~({pO`R4fT92RzBbmQIX_AgRqY@!pZXl!jIoK`!HYupW;+LnuS1L$_ zTtgCGPQ93Ofu*SVw(T z(Oi^u6HBb{iPd$g)GpXEFyhz#TkUmSCl^M?T1jDXk_ha!3>YqgByJM_D^=BRb>T_& z7V)Ai`)_%L(B3Rdj+HEuX-2fa-a6?>W(L@8@i*Oa^G5m?tm``C|DME->OjS>rljHq z0Srwm9z2Gmsi4)aFB3qXG25IhBDS=s zAl$ndOH32_tGB;|4>~ixv`KJ|&K&Hu!!ZRPXZ!Ug{6qxumFS(b3;f5hc@tcQl%6#o!5M>A%8oCyW-y?` zIpeEXY=ifEVh^SHMIkT_V{G~u49!u%ozuSl%S}j1N>aJ&kQ~8Qw`hz`#g@Cz#ZYuA zGy%yLU3g)2GG&s}Qz-Ab#y^P0UpH3?g0)}EHUk43OY_Tc>_RqN(n9JhMjR#E`xP7+5p|*@h=D1e@KGr z*8(f;&YKh&bFGt_&RMCQaZCqw8eR-r1hf>boqk{GxBerG6l?w(+v9uRqsvzoaJwdi zQ=bE`hZ_MM5um@P(?Yr$%FM7UwR5Rte`yb?6Y0YA@m!aUK zw>mnLb~Bm}5dcBtpFXyDk0>sL;Vj^E`*w|-f;qOVh`XIF>w9?n8_vomsqa$T7h@S3 zt#1g00Yj&J_j=dVX7)&vcA`!&Hw=z?$ujD-w4NYI)pX==NW=H&7fit7%CRR)=&T539tr5l z8zfGG96H2|=xr{``jSqYVL^-ACuiw0*z7qtp@3O}q*(`B;22$T>lf<+HV(eRtXt1e}b?Z6YW13?~h^|))Jj?H2C%D&xJ#;H99 zS+nw((e%{p=hwVOsmbj)TPy@sBupYgWPFZi);?X};#t^#&hVejU2RvPu&1W4eb zW8_uH18})oF8F1Xx$A_xAd%rS_ zAp7oi4MXTp!YE7?u4O1AnnkBAOJ9wU?zj8(7^8H>QJ?R!?5&B=tB3b!*;)7=mVF^v ziR2=j?$9z{^6P)4Pu&USF{PyNKtGyiRNN^FdpO&6WvFeVmkR%qwFy9MRuL?&EV{NGi7Y z93)fS5ZwM?1~uj@#MjD1mD@QIpbHB?oJj2BuQR!27;W2D*Es)Ve;XipZLOb~DkNxj zqzbvh!qAA<{gVH|XlTF?oh-EN-@rW{L0iNx-`W#~AM8_!iuq^(Vp6_U4+(#}>az2M?XSJGXk9*iW+ zYn0YtL8SneYoVfdE;L#C`kM2&c)g}CQtA@4agHI+TXrTE6QfJ$$20?{ph=FOcqz&9 zv>S-08J2NJChn68{%o-q4M%&R`Ph1zfTbwR&pp}2B5i{@8LcTq_}7~84cheWjDXkj z$;K%DJGND$6PM1YX;#He9M)7cQP;1xHjwS0Nkc;x(a+Ldig<6!hypH`&0y~Dx`!OD zVpt5C!7PYzJVmjCUe1Qjx>61jl7xwEELF>$l&ILYxn}9PO&>0UiH<{KAQY4FmZQ+E zoR_x6$&jdni9U@-&225niX&w&A?d;El=>9Ia4dU-&r((8we7S14cxgfD>5o)DijS! zX)&%B&&k6|d#Jmq*8Ce~+Sf21xfbn%TN46Tp-y>547m3=(QbnJ_ifFbcvt8^oqClw z0?G;=_`LKS7J*R*&@zjy(LLZ0yqW_b3h?`UD zeS=5~);=~V8#-1e` z_?CK(NbNbLaNMVRRPzK+Aycq_p4g`qwPywP{nhVm50%)Cg$LC(VI}u0o^D^^ucIU} zVWb%jQf5v~6S>k^&$si$GduP9#wykCXWJlXN5VNUcY|n;n&D1LQ+;gFO9)>#spxs*o*r)c z>DN`Afn-PL`i$swer-+-4uz?bc8fPDV*kLZ<{LbPd-P#(fUc?6D2MY^w6h%f&-TqH ztXRd2n>fh)gPvhgZ=dT6fS6+zQx-YkGA*ub)xsA?A?=`MpcfDvW6GhR5!YIdH?=AC zX5lxlq@Nc#dx5v_#(k|%L+AjmZJd=Pd71ymggyK64mx#lZ#w_&}{?hHlT&%+27Csh`Tf2m0(vi zxR$?3m{L#-0kSZ@k2Nn^t`00zM=ASu!77ajVFJM=R80v?iKh7Rlkm4Ap2`-(csoch zy#Sy6Dxg9we9z2}0XPQSV7eKr#>jFh2K0{)|L??+lp3E?`7YQ;c{w&lT1TY+;*4aC zDIHm%#^o%IL(_FqxOKy{lO?A4y6^L)IzImFZu%Lcr3Ye$0?h%p%D@oCatJermI?rn z{B}*Fji3Eky3kvh_h(a{AGD?|`nL8}-Z6ImIR9Z<$)*)M!QA5_)yS^#x}mv-_n46G zF#@e=U4$=}7LSk(pxZS}`C@yM6g=J`8){gu>;h0AHI9(kvc8?xFr?MM_TTYD?R8QL zc*lNRD`Si?>$D$+B^C3p*d+e@b?a(9;-jStF)nv){;TUFyM&WSoZDqEfy8xc?5Kz1 zPb2s~7kIAL%-J2n&KjP|W<^XEX1XF7QNp8RnkGu?%7OWZ-)!~?8HUQv~Ua{Y28;EyT5 zxWe#7W^f~x_Q2b-aMfps8>|x(A(4hSz}>gVBG8J5Gsto z*=sRE%(IT) zxGf8eh^%}BOZu3JM0IPDmcem%t-8KR8bO7E{_{RBuix-cf8#!0%|V_hv4w@Y2=zeo zJv$^5BxYpJ`fC0_{4Hll9O?4I&M7Y>I}6hZ3uzU28)1ixs5=ynRJ5R*Gpj5o2sOhv zndgac0=MK}j#r3U10d`j>Se;eQ7=M$m7#->u_7Yk(iiCH{k2`O8 zN{St#&>7OpzKCWkLam9}S+fk5VZuutyz$KI8xDKEB?!r~II)LD*jn(n8u!`b9B06d z$I8aB)QXc<0PXRw$^LSp6np12-B94WM#sFkR?KHQ4hs8)l7wi9-HN;C)9=wmZPT`DL!wv-8DAfu`#XSaXPmJ z&)QQR`amwD@(~$qr<-h<+F&I0({RJ`;y?aN@Es!e^&Bz2o_0vpjEqk7XVd4=y*Yrl z9>@~`z75nH!T?9u0RFD_74Qwuz3eECZ{xFpq=z}VRLy^h*mctIak(aYKM}f~T60yD zT1xe#OE3*9K$}l0KQXx|dx9yQM>qdQ+erNs@&{>&2kfnp2^2CcSsU=zz=3bs@NwV@ zK3l$R+0-_j-%I&VXp+~kjCu6UL+h`jC^$Y;T_Ot22dZTAlIY!0QMf6Jb6~s23_pb+ zB^3k)U@jzd6J|#2Q^2T0Qdbd(bX52%6V-dG3@OZ1)|~PATf!qi&2rx4E5qr1Q><=w zik?8}i;(D6l@q0H?3BGHuLM&qjOBIN*eP|a*L+!4lDb3bp>Eb=jS%Sm1*t#yM1U*x z%su%WM|tans&T$Ibm~Tbl$;K~%FudHlnR531`86zfQWT2(U05>;e8#7tpG*ilBlnVrfBpJK>2)_-(JCto!scr=$@r0cDy6jxr1c?IS;!Qhe%U_=(eT(yX zVh;uUFVe-^VkKTUDUP#7|5$_Yl@2hpAFX~#2(|<@wv``xZ||NBwRJyD#A$DZJA~vT z${7F%vHOd^<-cn)5WliNT9!y215m^)8ujA}%KMy5wjWVV@LakT+3CM5FSYpi2;-~M zH^;a1+##XiYTU_gY@Q>jd>mdpJ38dqvd#49eE&$?PW;=K$sawS!D%9;;k5Z1l=qY_yPZJoxr@PG$z;AVFF{eU(;YMDSdyHRf z6QyCJ(luh>7Gl&3@*n53V0Mp~-zB|Dlsk(k_o%~2xGYo(h1L$pYJi$Tbg*NYMRI(& zNho`!i3=#G_i4XNmL$)IfL5iYkCX2I*|GPejWYqV+jH(v0^)5w~El8+wP%373$9Wm#3DqKY}S}pRO?BWF-DZ zt+nuM{8vqH9Svo%V%VA%UBh@1mv~XKn}>4PumZz|4;dY#=>1G@B=dggFPPh{Z zr*RbUd}*4jmy$!0iO;KjaoUVp9yIvNcRUJ4q%^FzMuFBQzh03xw4xLfcu<_=$eA*7 zxyY*mW)kS*f6(G9Hg2g-<|`UU@-n1X8?s62(v^i2goiLKFRw-&Vcvt!ZPtHlXVbr7 z+e*oS(*zynS3L8X9x{N6iKXSszd*t@_{FQ41ju=0S3}SR!^)Gw&3+RO{r5DBm}|1b zuuXVfuabIF?n@mOW>XnTd)(2adjW3%{_dV?p>+3W;7kQde4@Qba6QoK=pmKw`qY_V zw2Bqow!M zaiM;rfj)k)=Xm>dI4?JEF1Lje6=)gGj`R_ngu&?C(#%RM%NMUgx`?O6OLgf-n^^JKO zjA=$aA?k!54Q1sa-)b_5L9AAZ36RSgyO^v=KJBfYUj=ABR>cP;oj<5Tjw+<{#{>7nYiGBQUgo zpFY8nYLmFWP+pir?mz^A7gj!pVi19L^JPzxcKPmq^%x0NOt{Voq1WS`ZUT=e&^sal zxs$;swQof;hR6+FM5w%@)U?bR!?+p(QW8?sb?JY-<6`GAcl+iy%}$th;3xXn()0t< zwmIp*D3UhEw`%L4tSJVCp#tS)g@Cg4cu!0q9ZPnH&G$3p5tQ23xbb>2iOlmF!=464jj`xT+1nl}M=>EVYAp({#uf_2{Y^x_(FwS*zYl*-6*y;UUoRBf3GLW z%2p(~;2AK~nF_Y61P;)VPB?mU*{bZXDFVwSq{IeLlAa%E{k4(}t* zEPn&v;woDv|LLmkoEeZ%Ny!>2{%m-+749V#JNF_z7bOf0dXMGHp&SYrGr=3+i&+76%sx@bArNZ}E)5w|q-91r?&LVai z=ln7O*Qf@ckE%tcj|*2exL4G=j@IMp1w`}CUZ@k-p8`BKP1B7`Kh%lv6O8^m;shh% zARiqin)j-v6AUpME(<&8)8{UngyW{WG)!b@otKzZGY2tol(kF;wWH{TWE~W)6VcEo zx|~EyK#L3L6BmDp%ZJy4Gzn!QJ}q4P{>5gt%Uu;@+UM4AXf^ z38MUUi?`3(PruJZ)wmc0f}_}=Fie?!MmzQwutBG6-lCOBrLH@DT3Twy2!OC*{LfcP zv*5nes@u1gj(rdLh=Yjh8ZFIx;C8se%z%tZ&saF8KPNiEmq%HQ@i8(c@2_+d0w7$U znH~6?DaJKJ@oAnv%nQY5GPY|?52#~^c2Cm5wa5?pHi)v*SS0+$(SU>SjfVEZj!YS? z_YVC9s_7K0LGes{iQZNVxZ7})rTs>j|DsuHAQI@?n{r_dR{|AC} B&$s{p literal 0 HcmV?d00001 diff --git a/mlx_streaming/prep/__init__.py b/mlx_streaming/prep/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mlx_streaming/prep/blob_layout.py b/mlx_streaming/prep/blob_layout.py new file mode 100644 index 0000000..78d8a1d --- /dev/null +++ b/mlx_streaming/prep/blob_layout.py @@ -0,0 +1,32 @@ +"""专家 blob 字节布局的单一真相源（v1 affine / v2 mxfp4）。 + +Segment 五元组 (proj, tensor, np_dtype_name, shape, nbytes)，与现有 +blob_loader._layout 的元组结构一致 → blob_loader 可直接消费、pack 取子集。 +""" +from typing import List, Tuple + +BLOB_V1_AFFINE = "expert_blob_v1" +BLOB_V2_MXFP4 = "expert_blob_v2_mxfp4" + +Segment = Tuple[str, str, str, Tuple[int, int], int] # proj, tensor, dtype, shape, nbytes + + +def _projs(hidden: int, inter: int) -> Tuple[Tuple[str, int, int], ...]: + return (("gate_proj", inter, hidden), ("up_proj", inter, hidden), ("down_proj", hidden, inter)) + + +def layout_for(fmt: str, hidden: int, inter: int, bits: int, group: int) -> Tuple[List[Segment], int]: + segs: List[Segment] = [] + for proj, out_d, in_d in _projs(hidden, inter): + words = in_d * bits // 32 + groups = in_d // group + segs.append((proj, "weight", "uint32", (out_d, words), out_d * words * 4)) + if fmt == BLOB_V1_AFFINE: + segs.append((proj, "scales", "uint16", (out_d, groups), out_d * groups * 2)) + segs.append((proj, "biases", "uint16", (out_d, groups), out_d * groups * 2)) + elif fmt == BLOB_V2_MXFP4: + segs.append((proj, "scales", "uint8", (out_d, groups), out_d * groups * 1)) + else: + raise ValueError(f"未知 blob format: {fmt}") + stride = sum(s[-1] for s in segs) + return segs, stride diff --git a/mlx_streaming/prep/download_retry.py b/mlx_streaming/prep/download_retry.py new file mode 100644 index 0000000..0b3d842 --- /dev/null +++ b/mlx_streaming/prep/download_retry.py @@ -0,0 +1,35 @@ +"""自动重试的断点续传下载：应对代理对大文件不稳定的情况。 + +反复调用 snapshot_download（huggingface_hub 会从 .incomplete 续传）， +每轮带超时；直到全部文件就绪或达到最大轮数。 +""" +import os +import sys +import time + +from huggingface_hub import snapshot_download + +REPO = os.environ.get("MODEL", "models/Qwen3-Next-80B-A3B-Instruct-MLX-8bit") +MAX_TRIES = int(os.environ.get("MAX_TRIES", "200")) +os.environ.setdefault("HF_HUB_DOWNLOAD_TIMEOUT", "30") + + +def main(): + for i in range(1, MAX_TRIES + 1): + try: + p = snapshot_download( + REPO, + allow_patterns=["*.json", "*.txt", "*.safetensors", "tokenizer*", "*.model"], + max_workers=1, + ) + print(f"DONE -> {p}", flush=True) + return + except Exception as e: + print(f"[try {i}] 失败，续传重试: {repr(e)[:160]}", flush=True) + time.sleep(3) + print("EXHAUSTED: 达到最大重试轮数仍未完成", flush=True) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/mlx_streaming/prep/extract_mtp.py b/mlx_streaming/prep/extract_mtp.py new file mode 100644 index 0000000..7454818 --- /dev/null +++ b/mlx_streaming/prep/extract_mtp.py @@ -0,0 +1,131 @@ +"""下载 Qwen3-Next 原版末分片并抽取/整理 MTP 权重为单文件。 + +MTP 全部权重在 model-00041-of-00041.safetensors(~3.9GB)。整理规则: + 1) 把 mlp.experts.{e}.{proj}.weight(512 个)stack 成 mlp.switch_mlp.{proj}.weight。 + 2) 对所有 RMSNorm 权重 +1.0(Qwen3-Next 用 zero-centered RMSNorm, + 与 mlx-lm sanitize 对主模型的处理一致;MTP 被 sanitize 过滤,故需自行补)。 +""" +import json +import os +import subprocess + +import mlx.core as mx + +# 与 mlx-lm qwen3_next.sanitize 一致的 norm 后缀(去掉 model.norm,因 MTP 用 mtp.norm) +_NORM_SUFFIXES = ( + ".input_layernorm.weight", + ".post_attention_layernorm.weight", + ".q_norm.weight", + ".k_norm.weight", +) + + +def _is_mtp_norm(key: str) -> bool: + if key.endswith(".pre_fc_norm_hidden.weight") or key.endswith( + ".pre_fc_norm_embedding.weight" + ): + return True + if key == "mtp.norm.weight": + return True + return any(key.endswith(sfx) for sfx in _NORM_SUFFIXES) + + +def bump_mtp_norms(weights: dict) -> dict: + out = {} + for k, v in weights.items(): + if _is_mtp_norm(k) and v.ndim == 1: + out[k] = v + 1.0 + else: + out[k] = v + return out + + +def stack_mtp_experts(weights: dict, num_experts: int) -> dict: + out = dict(weights) + prefix = "mtp.layers.0.mlp" + for proj in ("gate_proj", "up_proj", "down_proj"): + keys = [f"{prefix}.experts.{e}.{proj}.weight" for e in range(num_experts)] + if keys[0] not in out: + continue + stacked = mx.stack([out.pop(k) for k in keys]) + out[f"{prefix}.switch_mlp.{proj}.weight"] = stacked + return out + + +SHARD = "model-00041-of-00041.safetensors" +REPO = "Qwen/Qwen3-Next-80B-A3B-Instruct" +from mlx_streaming import config as _cfg +SHARD_DIR = os.environ.get("MTP_SHARD_DIR", "/tmp/qn_mtp_shard") # 临时分片，可留 /tmp +OUT_PATH = _cfg.mtp_out() # 默认 models/qn_mtp_weights.safetensors（持久） +CONFIG = _cfg.qn_config() # 默认 models/Qwen3-Next-80B-A3B-Instruct-MLX-8bit/config.json + +_DL_URL = ( + f"https://www.modelscope.cn/api/v1/models/{REPO}/repo" + f"?Revision=master&FilePath={SHARD}" +) + + +def _expected_size() -> int: + """从 ModelScope 文件列表 API 取 SHARD 的真实字节数。 + + 不能用 HEAD/content-length:API 重定向只回小 JSON,content-length 不可靠。 + """ + url = ( + f"https://www.modelscope.cn/api/v1/models/{REPO}/repo/files" + f"?Revision=master&Recursive=true" + ) + out = subprocess.run( + ["curl", "-sL", "--max-time", "60", url], + capture_output=True, text=True, + ).stdout + data = json.loads(out) + files = data.get("Data", {}).get("Files") or data.get("Data", {}).get("files") or [] + for f in files: + name = f.get("Name") or f.get("Path") or f.get("name") + if name and SHARD in str(name): + return int(f.get("Size") or f.get("size") or 0) + return 0 + + +def download_shard() -> str: + os.makedirs(SHARD_DIR, exist_ok=True) + out = os.path.join(SHARD_DIR, SHARD) + expect = _expected_size() + for attempt in range(1, 9): + cur = os.path.getsize(out) if os.path.exists(out) else 0 + if expect and cur == expect: + print(f"SKIP shard already complete ({cur}B)") + return out + print(f"download attempt {attempt}: have {cur}B / {expect}B") + subprocess.run( + ["curl", "-sL", "--max-time", "3600", + "--speed-limit", "51200", "--speed-time", "30", + _DL_URL, "-o", out], + ) + cur = os.path.getsize(out) if os.path.exists(out) else 0 + if expect and cur != expect: + raise RuntimeError(f"download failed: {cur}B != {expect}B") + return out + + +def extract(shard_path: str, config_path: str, out_path: str) -> None: + with open(config_path) as f: + cfg = json.load(f) + num_experts = cfg["num_experts"] + all_w = mx.load(shard_path) + mtp = {k: v for k, v in all_w.items() if k.startswith("mtp.")} + print(f"原始 mtp 张量数: {len(mtp)}") + mtp = stack_mtp_experts(mtp, num_experts) + mtp = bump_mtp_norms(mtp) + mx.eval(list(mtp.values())) + mx.save_safetensors(out_path, mtp) + print(f"已写出 {len(mtp)} 个张量 -> {out_path}") + + +def main(): + shard = download_shard() + extract(shard, CONFIG, OUT_PATH) + + +if __name__ == "__main__": + main() diff --git a/mlx_streaming/prep/pack_blob_from_experts.py b/mlx_streaming/prep/pack_blob_from_experts.py new file mode 100644 index 0000000..1750830 --- /dev/null +++ b/mlx_streaming/prep/pack_blob_from_experts.py @@ -0,0 +1,87 @@ +"""直接把 per-expert safetensors 打包成「每专家一个连续 blob」(按层一个文件)。 + +字节布局由 prep/blob_layout.py 统一描述（v1 affine / v2 mxfp4），与 blob_loader 完全一致： +按 _split_meta.json 的 blob_format 选段表，每专家依次写各 proj 的 [weight, scales(, biases)]。 +- weight: uint32；v1 的 scales/biases 为 bf16 按 uint16 原始 2 字节；v2 mxfp4 的 scales 为 uint8。 + +环境变量：EXPERT_DIR(源 per-expert) / BLOB_DIR(输出) / BITS / GROUP / LAYERS(逗号或 all) +""" +import json +import os + +import mlx.core as mx +import numpy as np + +from mlx_streaming.prep.blob_layout import layout_for, BLOB_V1_AFFINE, BLOB_V2_MXFP4 + +EXPERT_DIR = os.environ.get("EXPERT_DIR", "/tmp/qwen3_next_experts_8bit_g128") +OUT = os.environ.get("BLOB_DIR", "/tmp/cb_8bit_blob") +BITS = int(os.environ.get("BITS", "8")) +GROUP = int(os.environ.get("GROUP", "128")) + + +def _meta(): + m = json.load(open(os.path.join(EXPERT_DIR, "_split_meta.json"))) + d = m["dims"] + fmt = m.get("blob_format", BLOB_V1_AFFINE) + return (int(d["num_experts"]), int(d["hidden"]), int(d["moe_intermediate"]), + fmt, d.get("quant_mode", "affine")) + + +def _raw_bytes(arr) -> bytes: + """uint32 → 4B/元素；uint8 → 1B/元素；其余(affine 的 bf16 scales/biases) → uint16 2B/元素。""" + if arr.dtype == mx.uint32: + return np.array(arr, copy=False).tobytes() + if arr.dtype == mx.uint8: + return np.array(arr, copy=False).tobytes() + return np.array(arr.view(mx.uint16), copy=False).tobytes() + + +def pack_layer(layer, num_experts, segs, stride, fmt, quant_mode) -> None: + out_path = os.path.join(OUT, f"layer{layer:02d}.blob") + with open(out_path, "wb") as f: + for e in range(num_experts): + w = mx.load(os.path.join(EXPERT_DIR, f"layer{layer:02d}_expert{e:03d}.safetensors")) + for proj, tensor, dt, shape, nb in segs: + b = _raw_bytes(w[f"{proj}.{tensor}"]) + assert len(b) == nb, f"L{layer} e{e} {proj}.{tensor}: {len(b)}!={nb}" + f.write(b) + assert os.path.getsize(out_path) == stride * num_experts + index = {"format": fmt, "quant_mode": quant_mode, "layer": layer, + "num_experts": num_experts, "stride": stride, + "page_aligned": stride % 16384 == 0, + "segments": [{"proj": p, "tensor": t, "nbytes": n} for p, t, _, _, n in segs]} + with open(os.path.join(OUT, f"layer{layer:02d}.blob.index.json"), "w") as f: + json.dump(index, f, ensure_ascii=False, indent=2) + + +def _resolve_layers(spec: str, num_experts: int) -> list: + spec = spec.strip() + if spec.lower() == "all": + layers = set() + for name in os.listdir(EXPERT_DIR): + if name.startswith("layer") and name.endswith("_expert000.safetensors"): + layers.add(int(name[5:7])) + return sorted(layers) + return [int(x) for x in spec.split(",") if x.strip()] + + +def main(): + os.makedirs(OUT, exist_ok=True) + num_experts, hidden, inter, fmt, quant_mode = _meta() + segs, stride = layout_for(fmt, hidden, inter, BITS, GROUP) + layers = _resolve_layers(os.environ.get("LAYERS", "all"), num_experts) + for i, L in enumerate(layers): + pack_layer(L, num_experts, segs, stride, fmt, quant_mode) + print(f" layer {L} packed ({i+1}/{len(layers)})", flush=True) + summary = {"format": fmt, "quant_mode": quant_mode, "stride": stride, + "page_aligned": stride % 16384 == 0, "num_experts": num_experts, + "layers": layers, "bits": BITS, "group_size": GROUP} + with open(os.path.join(OUT, "blob_index.json"), "w") as f: + json.dump(summary, f, ensure_ascii=False, indent=2) + print(json.dumps({"out": OUT, "n_layers": len(layers), "stride_bytes": stride, + "page_aligned": stride % 16384 == 0, "format": fmt}, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/mlx_streaming/prep/pack_compute_buffers.py b/mlx_streaming/prep/pack_compute_buffers.py new file mode 100644 index 0000000..a8f2549 --- /dev/null +++ b/mlx_streaming/prep/pack_compute_buffers.py @@ -0,0 +1,83 @@ +"""把单层单 projection 打包成连续 compute buffers。 + +输出三个大 buffer: +- layer43.gate_proj.weight.bin +- layer43.gate_proj.scales.bin +- layer43.gate_proj.biases.bin + +这个格式用于验证“整层少数大 buffer + custom qlinear kernel 直接按 expert_id 读取”。 +""" +import argparse +import json +import os + +from mlx_streaming.prep.pack_expert_ranges import read_safetensors_payloads + + +def pack_projection(src_dir: str, out_dir: str, layer: int, proj: str, num_experts: int) -> dict: + os.makedirs(out_dir, exist_ok=True) + tensors = ["weight", "scales", "biases"] + handles = { + name: open(os.path.join(out_dir, f"layer{layer:02d}.{proj}.{name}.bin"), "wb") + for name in tensors + } + meta = { + "format": "mlx_streaming_compute_buffer_v1", + "src_dir": src_dir, + "layer": layer, + "proj": proj, + "num_experts": num_experts, + "tensors": {}, + } + try: + for expert in range(num_experts): + path = os.path.join(src_dir, f"layer{layer:02d}_expert{expert:03d}.safetensors") + payloads = {rec["key"]: rec for rec in read_safetensors_payloads(path)} + for name in tensors: + key = f"{proj}.{name}" + rec = payloads[key] + f = handles[name] + offset = f.tell() + f.write(rec["payload"]) + entry = meta["tensors"].setdefault(name, { + "dtype": rec["dtype"], + "shape_per_expert": rec["shape"], + "nbytes_per_expert": len(rec["payload"]), + "file": f"layer{layer:02d}.{proj}.{name}.bin", + "offsets": [], + }) + entry["offsets"].append(offset) + finally: + for f in handles.values(): + f.close() + meta_path = os.path.join(out_dir, f"layer{layer:02d}.{proj}.index.json") + with open(meta_path, "w") as f: + json.dump(meta, f, ensure_ascii=False, indent=2) + return meta + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--src", default=os.environ.get("EXPERT_DIR", "")) + ap.add_argument("--out", default=os.environ.get("COMPUTE_BUFFER_DIR", "")) + ap.add_argument("--layer", type=int, default=int(os.environ.get("LAYER", "43"))) + ap.add_argument("--proj", default=os.environ.get("PROJ", "gate_proj")) + args = ap.parse_args() + if not args.src: + raise SystemExit("--src / EXPERT_DIR required") + out_dir = args.out or os.path.join(args.src, "compute_buffers") + with open(os.path.join(args.src, "_split_meta.json")) as f: + model_meta = json.load(f) + num_experts = int(model_meta["dims"]["num_experts"]) + meta = pack_projection(args.src, out_dir, args.layer, args.proj, num_experts) + print(json.dumps({ + "out_dir": out_dir, + "layer": args.layer, + "proj": args.proj, + "num_experts": num_experts, + "files": {k: v["file"] for k, v in meta["tensors"].items()}, + }, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/mlx_streaming/prep/pack_expert_bundles.py b/mlx_streaming/prep/pack_expert_bundles.py new file mode 100644 index 0000000..1d4edbd --- /dev/null +++ b/mlx_streaming/prep/pack_expert_bundles.py @@ -0,0 +1,70 @@ +"""把 per-expert safetensors 打包成 per-layer bundle。 + +输入目录: + layer00_expert000.safetensors + layer00_expert001.safetensors + ... + +输出目录: + layer_bundles/layer00.safetensors + +bundle 内 key: + expert000.gate_proj.weight + expert000.gate_proj.scales + ... + +运行时设置 EXPERT_BUNDLE=1 即可优先读取 bundle。 +""" +import json +import os +import time + +import mlx.core as mx + +SRC_DIR = os.environ.get("EXPERT_DIR", "/tmp/qwen3_next_experts") +OUT_DIR = os.environ.get("EXPERT_BUNDLE_DIR", os.path.join(SRC_DIR, "layer_bundles")) + + +def _pack_layer(src_dir: str, out_dir: str, layer: int, num_experts: int) -> str: + out = {} + for e in range(num_experts): + path = os.path.join(src_dir, f"layer{layer:02d}_expert{e:03d}.safetensors") + rec = mx.load(path) + prefix = f"expert{e:03d}." + for k, v in rec.items(): + out[prefix + k] = v + os.makedirs(out_dir, exist_ok=True) + out_path = os.path.join(out_dir, f"layer{layer:02d}.safetensors") + mx.save_safetensors(out_path, out) + return out_path + + +def main(): + t0 = time.perf_counter() + with open(os.path.join(SRC_DIR, "_split_meta.json")) as f: + meta = json.load(f) + layers = [int(x) for x in meta["moe_layers"]] + num_experts = int(meta["dims"]["num_experts"]) + os.makedirs(OUT_DIR, exist_ok=True) + for i, layer in enumerate(layers): + out_path = _pack_layer(SRC_DIR, OUT_DIR, layer, num_experts) + print(f" {i + 1}/{len(layers)} layer{layer:02d} -> {out_path} " + f"({round(time.perf_counter() - t0, 1)}s)", flush=True) + bundle_meta = { + "src_dir": SRC_DIR, + "out_dir": OUT_DIR, + "layers": layers, + "num_experts": num_experts, + "format": "expert{EEE}.{tensor_key}", + } + with open(os.path.join(OUT_DIR, "_bundle_meta.json"), "w") as f: + json.dump(bundle_meta, f, ensure_ascii=False, indent=2) + print(json.dumps({ + "out_dir": OUT_DIR, + "layers": len(layers), + "elapsed_s": round(time.perf_counter() - t0, 1), + }, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/mlx_streaming/prep/pack_expert_ranges.py b/mlx_streaming/prep/pack_expert_ranges.py new file mode 100644 index 0000000..03567b7 --- /dev/null +++ b/mlx_streaming/prep/pack_expert_ranges.py @@ -0,0 +1,135 @@ +"""把 per-expert safetensors 的原始 payload 打包成 per-layer range pack。 + +这个格式只用于 de-risk：避免运行时解析大量小 safetensors header，先验证 +“按 expert byte range 读取”是否比多个 `mx.load` 更快。 +""" +import argparse +import json +import os +import struct +import time +from pathlib import Path + +ALIGN = 64 + + +def _align(n: int, align: int = ALIGN) -> int: + rem = n % align + return n if rem == 0 else n + (align - rem) + + +def read_safetensors_payloads(path: str) -> list[dict]: + """读取 safetensors header 和每个 tensor 的原始 payload bytes。""" + with open(path, "rb") as f: + header_len = struct.unpack(" dict: + """打包单层所有专家，返回 index dict。""" + os.makedirs(out_dir, exist_ok=True) + pack_path = os.path.join(out_dir, f"layer{layer:02d}.pack") + index_path = os.path.join(out_dir, f"layer{layer:02d}.index.json") + tsv_path = os.path.join(out_dir, f"layer{layer:02d}.idx") + tensors = [] + experts = [] + with open(pack_path, "wb") as pack: + for expert in range(num_experts): + src = os.path.join(src_dir, f"layer{layer:02d}_expert{expert:03d}.safetensors") + expert_start = _align(pack.tell()) + if expert_start > pack.tell(): + pack.write(b"\0" * (expert_start - pack.tell())) + for rec in read_safetensors_payloads(src): + offset = _align(pack.tell()) + if offset > pack.tell(): + pack.write(b"\0" * (offset - pack.tell())) + payload = rec["payload"] + pack.write(payload) + tensors.append({ + "layer": layer, + "expert_id": expert, + "key": rec["key"], + "dtype": rec["dtype"], + "shape": rec["shape"], + "offset": offset, + "nbytes": len(payload), + }) + expert_end = pack.tell() + experts.append({ + "layer": layer, + "expert_id": expert, + "offset": expert_start, + "nbytes": expert_end - expert_start, + }) + index = { + "format": "mlx_streaming_expert_pack_v1", + "alignment": ALIGN, + "layer": layer, + "num_experts": num_experts, + "pack": os.path.basename(pack_path), + "experts": experts, + "tensors": tensors, + } + with open(index_path, "w") as f: + json.dump(index, f, ensure_ascii=False, indent=2) + with open(tsv_path, "w") as f: + f.write("kind\tlayer\texpert_id\tkey\tdtype\tshape\toffset\tnbytes\n") + for rec in experts: + f.write( + f"EXPERT\t{rec['layer']}\t{rec['expert_id']}\t*\t*\t*\t" + f"{rec['offset']}\t{rec['nbytes']}\n" + ) + for rec in tensors: + shape = ",".join(str(x) for x in rec["shape"]) + f.write( + f"TENSOR\t{rec['layer']}\t{rec['expert_id']}\t{rec['key']}\t" + f"{rec['dtype']}\t{shape}\t{rec['offset']}\t{rec['nbytes']}\n" + ) + return index + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--src", default=os.environ.get("EXPERT_DIR", "/tmp/qwen3_next_experts")) + ap.add_argument("--out", default=os.environ.get("EXPERT_PACK_DIR")) + ap.add_argument("--layers", default=os.environ.get("LAYERS", "")) + args = ap.parse_args() + src = args.src + out = args.out or os.path.join(src, "layer_packs") + with open(os.path.join(src, "_split_meta.json")) as f: + meta = json.load(f) + all_layers = [int(x) for x in meta["moe_layers"]] + layers = [int(x) for x in args.layers.split(",") if x.strip()] if args.layers else all_layers + num_experts = int(meta["dims"]["num_experts"]) + t0 = time.perf_counter() + for i, layer in enumerate(layers): + pack_layer(src, out, layer, num_experts) + print(f" {i + 1}/{len(layers)} layer{layer:02d} ({round(time.perf_counter() - t0, 1)}s)", flush=True) + with open(os.path.join(out, "_pack_meta.json"), "w") as f: + json.dump({ + "src": src, + "out": out, + "layers": layers, + "num_experts": num_experts, + "format": "mlx_streaming_expert_pack_v1", + }, f, ensure_ascii=False, indent=2) + print(json.dumps({"out": out, "layers": len(layers)}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/mlx_streaming/prep/repack_expert_blobs.py b/mlx_streaming/prep/repack_expert_blobs.py new file mode 100644 index 0000000..5c5413a --- /dev/null +++ b/mlx_streaming/prep/repack_expert_blobs.py @@ -0,0 +1,92 @@ +"""把 compute buffer 重打包成"每专家一个连续 blob"格式（按层一个文件）。 + +blob 内顺序：gate[w,s,b] + up[w,s,b] + down[w,s,b]，正好 864KB（= 54×16KB 页对齐）。 +读一个专家 = 1 次 pread(stride, e*stride)，而不是当前的 9 次散读。 + +环境变量：COMPUTE_BUFFER_DIR(源)、BLOB_DIR(输出)、LAYERS(逗号分隔，默认 15,25) +""" +import json +import os + +import numpy as np + +HIDDEN = 2048 +INTER = 512 +GROUP = 128 +BITS = 2 +NUM_EXPERTS = 512 +SRC = os.environ.get("COMPUTE_BUFFER_DIR", "/tmp/cb_2bit_g128") +OUT = os.environ.get("BLOB_DIR", "/tmp/cb_2bit_blob") +PROJS = (("gate_proj", INTER, HIDDEN), ("up_proj", INTER, HIDDEN), ("down_proj", HIDDEN, INTER)) + + +def _layout(): + """返回 [(proj, tensor, nbytes_per_expert), ...] 和单专家总字节。""" + segs = [] + for proj, out_dim, in_dim in PROJS: + words = in_dim * BITS // 32 + groups = in_dim // GROUP + segs.append((proj, "weight", out_dim * words * 4)) + segs.append((proj, "scales", out_dim * groups * 2)) + segs.append((proj, "biases", out_dim * groups * 2)) + return segs, sum(s[2] for s in segs) + + +def repack_layer(layer: int) -> dict: + os.makedirs(OUT, exist_ok=True) + segs, stride = _layout() + # 源 .bin 以 uint8 原始字节映射，按专家字节偏移切片 + raw = {} + for proj, _, _ in PROJS: + base = os.path.join(SRC, f"layer{layer:02d}.{proj}") + for tensor in ("weight", "scales", "biases"): + raw[(proj, tensor)] = np.memmap(f"{base}.{tensor}.bin", dtype=np.uint8, mode="r") + out_path = os.path.join(OUT, f"layer{layer:02d}.blob") + with open(out_path, "wb") as f: + for e in range(NUM_EXPERTS): + for proj, tensor, nb in segs: + buf = raw[(proj, tensor)] + f.write(buf[e * nb:(e + 1) * nb].tobytes()) + assert os.path.getsize(out_path) == stride * NUM_EXPERTS + index = { + "format": "expert_blob_v1", + "layer": layer, + "num_experts": NUM_EXPERTS, + "stride": stride, + "page_aligned": stride % 16384 == 0, + "segments": [{"proj": p, "tensor": t, "nbytes": n} for p, t, n in segs], + } + with open(os.path.join(OUT, f"layer{layer:02d}.blob.index.json"), "w") as f: + json.dump(index, f, ensure_ascii=False, indent=2) + return index + + +def _resolve_layers(spec: str) -> list[int]: + """LAYERS=all → 读源 compute buffer 现有的全部层；否则按逗号解析。""" + spec = spec.strip() + if spec.lower() == "all": + layers = [] + for name in os.listdir(SRC): + if name.endswith(".gate_proj.weight.bin") and name.startswith("layer"): + layers.append(int(name[len("layer"):len("layer") + 2])) + return sorted(set(layers)) + return [int(x) for x in spec.split(",") if x.strip()] + + +def main(): + layers = _resolve_layers(os.environ.get("LAYERS", "15,25")) + _, stride = _layout() + for L in layers: + repack_layer(L) + # 汇总 index，供 loader 校验/发现 + summary = {"format": "expert_blob_v1", "stride": stride, + "page_aligned": stride % 16384 == 0, "num_experts": NUM_EXPERTS, + "layers": layers} + with open(os.path.join(OUT, "blob_index.json"), "w") as f: + json.dump(summary, f, ensure_ascii=False, indent=2) + print(json.dumps({"out": OUT, "n_layers": len(layers), "stride_bytes": stride, + "page_aligned": stride % 16384 == 0}, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/mlx_streaming/prep/split_experts.py b/mlx_streaming/prep/split_experts.py new file mode 100644 index 0000000..6326de3 --- /dev/null +++ b/mlx_streaming/prep/split_experts.py @@ -0,0 +1,69 @@ +"""路线 B 离线工具：把模型里堆叠的 switch_mlp 专家权重按专家拆成 per-expert 小文件。 + +拆分后每个文件 layer{L:02d}_expert{E:03d}.safetensors 含扁平 dict： + gate_proj.weight / gate_proj.scales / gate_proj.biases + up_proj.weight / up_proj.scales / up_proj.biases + down_proj.weight / down_proj.scales / down_proj.biases +（非量化模型则只有 .weight，可能还有 .bias） + +一次拆分、逐专家物化，全程低内存。 +""" +import os +import sys +import json + +import mlx.core as mx + +PROJ_NAMES = ["gate_proj", "up_proj", "down_proj"] + + +def split_switch_glu(switch_glu, out_dir: str, layer: int) -> int: + """把一个 SwitchGLU 的三组 SwitchLinear 沿专家维拆成 per-expert 文件。返回专家数。""" + os.makedirs(out_dir, exist_ok=True) + E = switch_glu.gate_proj.num_experts + for e in range(E): + d = {} + for proj_name in PROJ_NAMES: + proj = getattr(switch_glu, proj_name) + for pname, p in proj.parameters().items(): + if isinstance(p, mx.array) and p.ndim >= 1 and p.shape[0] == E: + d[f"{proj_name}.{pname}"] = p[e] + mx.eval(d) # 只物化这一个专家 + path = os.path.join(out_dir, f"layer{layer:02d}_expert{e:03d}.safetensors") + mx.save_safetensors(path, d) + return E + + +def split_model(model_path: str, out_dir: str) -> dict: + """加载模型（lazy）并把所有 MoE 层的专家拆到 out_dir。返回 {dims/统计}。""" + from mlx_lm import load + model, _ = load(model_path, lazy=True) + os.makedirs(out_dir, exist_ok=True) + moe_layers = [] + dims = None + for l, layer in enumerate(model.layers): + mlp = getattr(layer, "mlp", None) + if mlp is not None and hasattr(mlp, "switch_mlp") and hasattr(mlp, "gate"): + sm = mlp.switch_mlp + split_switch_glu(sm, out_dir, l) + moe_layers.append(l) + if dims is None: + gp = sm.gate_proj + dims = { + "hidden": gp.input_dims, + "moe_intermediate": gp.output_dims, + "num_experts": gp.num_experts, + "group_size": getattr(gp, "group_size", None), + "bits": getattr(gp, "bits", None), + } + meta = {"out_dir": out_dir, "moe_layers": moe_layers, "dims": dims} + with open(os.path.join(out_dir, "_split_meta.json"), "w") as f: + json.dump(meta, f, ensure_ascii=False, indent=2) + return meta + + +if __name__ == "__main__": + mp = sys.argv[1] + od = sys.argv[2] if len(sys.argv) > 2 else "/tmp/mlx_qwen3_experts" + m = split_model(mp, od) + print(json.dumps(m, ensure_ascii=False, indent=2)) diff --git a/mlx_streaming/runtime/__init__.py b/mlx_streaming/runtime/__init__.py new file mode 100644 index 0000000..eaf1dad --- /dev/null +++ b/mlx_streaming/runtime/__init__.py @@ -0,0 +1 @@ +"""生产推理入口：基线/流式/投机解码等 run_* 启动脚本。""" diff --git a/mlx_streaming/runtime/run_baseline.py b/mlx_streaming/runtime/run_baseline.py new file mode 100644 index 0000000..17deeb5 --- /dev/null +++ b/mlx_streaming/runtime/run_baseline.py @@ -0,0 +1,42 @@ +"""全量驻留基线：正常加载 + 生成，记录 RSS/峰值/decode 速度。""" +import os +import time +import json + +import mlx.core as mx +from mlx_lm import load, generate + +from mlx_streaming.core.mem import snapshot, reset_peak + +MODEL = os.environ.get("MODEL", "models/Qwen3-Next-80B-A3B-Instruct-MLX-8bit") +PROMPT = os.environ.get("PROMPT", "用三句话解释什么是混合专家模型。") +MAXTOK = int(os.environ.get("MAXTOK", "128")) + + +def main(): + reset_peak() + t0 = time.perf_counter() + model, tok = load(MODEL) # 默认全量加载 + mx.eval(model.parameters()) # 强制全部驻留，作为对照上界 + load_done = snapshot() + t1 = time.perf_counter() + + text = generate(model, tok, prompt=PROMPT, max_tokens=MAXTOK, verbose=False) + t2 = time.perf_counter() + after = snapshot() + + out = { + "mode": "baseline_resident", + "model": MODEL, + "load_s": round(t1 - t0, 2), + "gen_s": round(t2 - t1, 2), + "tok_per_s": round(MAXTOK / (t2 - t1), 2), + "rss_gb_after_load": round(load_done.rss_bytes / 1e9, 2), + "rss_gb_after_gen": round(after.rss_bytes / 1e9, 2), + "mlx_peak_gb": round(after.mlx_peak_bytes / 1e9, 2), + } + print(json.dumps(out, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/mlx_streaming/runtime/run_mmap.py b/mlx_streaming/runtime/run_mmap.py new file mode 100644 index 0000000..77ee85a --- /dev/null +++ b/mlx_streaming/runtime/run_mmap.py @@ -0,0 +1,69 @@ +"""路线 A：lazy（+ 若支持则 mmap）加载，可选设常驻/缓存上限，量内存与速度。 + +环境探针结论（mlx-lm 0.31.3）：load 支持 lazy，但不支持 use_mmap。 +本脚本用 _filtered_load_kwargs 只传该版本真正支持的参数，缺失的自动剔除。 +""" +import os +import time +import json +import inspect + +import mlx.core as mx +from mlx_lm import load, generate + +from mlx_streaming.core.mem import snapshot, reset_peak, clear_cache + +MODEL = os.environ.get("MODEL", "models/Qwen3-Next-80B-A3B-Instruct-MLX-8bit") +PROMPT = os.environ.get("PROMPT", "用三句话解释什么是混合专家模型。") +MAXTOK = int(os.environ.get("MAXTOK", "128")) +WIRED_GB = os.environ.get("WIRED_GB") # 例如 "6"；空=不设 +CACHE_GB = os.environ.get("CACHE_GB") # 例如 "1"；空=不设 + + +def _maybe(fn_name, val_bytes): + fn = getattr(mx, fn_name, None) or getattr(getattr(mx, "metal", object()), fn_name, None) + if fn and val_bytes is not None: + prev = fn(int(val_bytes)) + print(f"{fn_name}({int(val_bytes)}) 旧值={prev}") + + +def _filtered_load_kwargs(): + sig = inspect.signature(load) + want = {"lazy": True, "use_mmap": True} + return {k: v for k, v in want.items() if k in sig.parameters} + + +def main(): + if WIRED_GB: + _maybe("set_wired_limit", float(WIRED_GB) * 1e9) + if CACHE_GB: + _maybe("set_cache_limit", float(CACHE_GB) * 1e9) + + reset_peak() + kw = _filtered_load_kwargs() + print("load kwargs:", kw) + t0 = time.perf_counter() + model, tok = load(MODEL, **kw) # 不强制 eval 全部参数！ + t1 = time.perf_counter() + after_load = snapshot() # 关键：此时 RSS 应远低于基线（专家还没换入） + + text = generate(model, tok, prompt=PROMPT, max_tokens=MAXTOK, verbose=False) + t2 = time.perf_counter() + clear_cache() + after_gen = snapshot() + + out = { + "mode": "mmap_lazy", "model": MODEL, + "wired_gb": WIRED_GB, "cache_gb": CACHE_GB, + "load_s": round(t1 - t0, 2), "gen_s": round(t2 - t1, 2), + "tok_per_s": round(MAXTOK / (t2 - t1), 2), + "rss_gb_after_load": round(after_load.rss_bytes / 1e9, 2), + "rss_gb_after_gen": round(after_gen.rss_bytes / 1e9, 2), + "mlx_active_gb_after_gen": round(after_gen.mlx_active_bytes / 1e9, 2), + "mlx_peak_gb": round(after_gen.mlx_peak_bytes / 1e9, 2), + } + print(json.dumps(out, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/mlx_streaming/runtime/run_mtp_spec.py b/mlx_streaming/runtime/run_mtp_spec.py new file mode 100644 index 0000000..746d850 --- /dev/null +++ b/mlx_streaming/runtime/run_mtp_spec.py @@ -0,0 +1,409 @@ +"""真实 80B MTP 自投机基准 + 与非投机贪婪逐 token 一致性校验。 + +环境变量:K / MAXTOK / PROMPT / QN_CONFIG / MTP_OUT(其余主模型路径见 validate_mtp)。 +""" +import json +import os +import statistics +import time + +import mlx.core as mx +from mlx_lm.models.qwen3_next import ModelArgs + +from mlx_streaming.core.moe import native_moe +from mlx_streaming.core.mem import snapshot, reset_peak +from mlx_streaming.mtp.drafter import MTPDrafter +from mlx_streaming.mtp.generate import forward_with_hidden, mtp_generate, prefill_chunked +from mlx_streaming.mtp.qwen3_next_mtp import load_mtp +from mlx_streaming.model_builder import build_streaming_model + +from mlx_streaming import config as _cfg +QN_CONFIG = _cfg.qn_config() +MTP_OUT = _cfg.mtp_out() +PROMPT = os.environ.get("PROMPT", "用三句话解释什么是混合专家模型。") +MAXTOK = int(os.environ.get("MAXTOK", "96")) +K = int(os.environ.get("K", "3")) +PIN_HOT = int(os.environ.get("PIN_HOT", "0")) +PIN_CAL_TOK = int(os.environ.get("PIN_CAL_TOK", "32")) +# 稳态测速:warmup 跑满 MAXTOK(热 baseline+spec 两条路径的 Metal kernel 与常驻专家池/预取), +# 再各重复 REPEAT 次取中位数,避免冷启动编译/补池污染绝对 tok/s。WARMUP_TOK=0 关 warmup。 +WARMUP_TOK = int(os.environ.get("WARMUP_TOK", str(MAXTOK))) +REPEAT = int(os.environ.get("REPEAT", "3")) + + +def _spec_once(model, drafter, tok, n, k): + ids, stats = mtp_generate(model, drafter, tok, + mx.array([tok.encode(PROMPT)]), + n, K=k, ids_mode=True, profile=True) + tps = round(stats["tokens"] / stats["wall_s"], 2) + return ids, stats, tps + + +def _baseline_greedy(model, tok, prompt, n): + cache = model.make_cache() + ids = mx.array([tok.encode(prompt)]) + t0 = time.perf_counter() + # prefill 分块:把整段 prompt 的激活峰值压到与 decode 同稳态(见 config.prefill_chunk)。 + logits, _ = prefill_chunked(model, ids, cache) + _dump_margin = bool(os.environ.get("DUMP_MARGIN")) + out = [] + for _ in range(n): + lg = logits[:, -1, :] + nxt = int(mx.argmax(lg)) + out.append(nxt) + if _dump_margin: + # 诊断:每步 top-2 logit 与差值,用于判定路径间发散是「FP 近平局」还是「真错」。 + v = lg.reshape(-1) + top2 = mx.argpartition(-v, 2)[:2] + t2 = [int(x) for x in top2.tolist()] + vv = {t: float(v[t]) for t in t2} + st = sorted(t2, key=lambda t: -vv[t]) + print(f"MARGIN step={len(out)-1} top1={st[0]}({vv[st[0]]:.5f}) " + f"top2={st[1]}({vv[st[1]]:.5f}) gap={vv[st[0]]-vv[st[1]]:.6f}", flush=True) + cur = mx.array([[nxt]]) + mx.eval(cur) + if len(out) >= n: + break + logits, _ = forward_with_hidden(model, cur, cache) + return out, round(n / (time.perf_counter() - t0), 2) + + +def main(): + reset_peak() + model, tok, store = build_streaming_model() + with open(QN_CONFIG) as f: + args = ModelArgs.from_dict(json.load(f)) + mtp = load_mtp(args, MTP_OUT, quantize=True) + mtp.embed_tokens = model.model.embed_tokens # 共享主模型 embedding + drafter = MTPDrafter(mtp, model.lm_head) + + # warmup:同时跑 baseline + spec 两条路径,编译 Metal kernel(含 multistate/batch verify) + # 并把专家常驻池/预取热起来,确保后续测的是稳态而非冷启动。默认 warmup=MAXTOK 跑满全长。 + if WARMUP_TOK > 0: + _baseline_greedy(model, tok, PROMPT, WARMUP_TOK) + _spec_once(model, drafter, tok, WARMUP_TOK, K) + + # ---- baseline 稳态:重复 REPEAT 次取中位数;最后一次清零统计供命中率口径 ---- + base = None + base_tps_runs = [] + for r in range(REPEAT): + if r == REPEAT - 1: + store.reset_stats() + base, bt = _baseline_greedy(model, tok, PROMPT, MAXTOK) + base_tps_runs.append(bt) + base_tps = statistics.median(base_tps_runs) + base_miss, base_hit = store.misses, store.hits + base_prefetch_loads = store._resident.prefetch_loads + base_prefetch_hits = store._resident.prefetch_hits + + # 可选:baseline 之后、spec 之前校准每层热专家，并预取/钉入 resident pool。 + # 这样 disk_load_ratio 的分母仍是未 pin baseline，能直接衡量 pin 是否压低 spec miss。 + if PIN_HOT > 0: + store.record = True + _baseline_greedy(model, tok, PROMPT, PIN_CAL_TOK) + for li in store.recorded_layers(): + store.pin(li, store.hot(li, PIN_HOT)) + store.record = False + store.reset_stats() + + # 诊断:开 handler 触发时刻探针(仅覆盖 spec 阶段,enable 会清零旧日志)。 + _hprof = bool(os.environ.get("STAGING_HPROF")) + if _hprof: + from mlx_streaming import native_moe_ext as _Nhp + _Nhp.staging_hprof_enable(True) + + # ---- spec 稳态:重复 REPEAT 次取中位数;最后一次清零统计 + reset_peak + 保留其 stats ---- + ids, stats, spec_tps_runs = None, None, [] + for r in range(REPEAT): + if r == REPEAT - 1: + store.reset_stats() + reset_peak() + from mlx_streaming.core.profiling import tprof_reset, union_reset + tprof_reset() # 探针只统计最终测量轮(与命中率/内存口径一致) + union_reset() # 并集专家数也只统计最终轮 + ids, stats, st = _spec_once(model, drafter, tok, MAXTOK, K) + spec_tps_runs.append(st) + spec_tps = statistics.median(spec_tps_runs) + spec_miss, spec_hit = store.misses, store.hits + spec_prefetch_loads = store._resident.prefetch_loads + spec_prefetch_hits = store._resident.prefetch_hits + after = snapshot() + proj = stats.get("proj_no_replay_tps", 0.0) + result = { + "K": K, + "max_tokens": MAXTOK, + "warmup_tok": WARMUP_TOK, + "repeat": REPEAT, + "exact_match": ids == base, + "n_mismatch": sum(1 for a, b in zip(ids, base) if a != b), + "avg_accept_len": stats["avg_accept_len"], + "steps": stats["steps"], + "verify_mode": stats.get("verify_mode"), + "direct_commits": stats.get("direct_commits"), + "fallback_replays": stats.get("fallback_replays"), + "replayed_tokens": stats.get("replayed_tokens"), + "spec_tok_per_s": spec_tps, + "baseline_tok_per_s": base_tps, + "speedup": round(spec_tps / max(base_tps, 1e-6), 2), + "spec_tps_runs": spec_tps_runs, + "baseline_tps_runs": base_tps_runs, + "spec_tps_minmax": [min(spec_tps_runs), max(spec_tps_runs)], + # 分段计时与「重放免费」投机上限 + "t_draft_s": stats.get("t_draft_s"), + "t_snap_s": stats.get("t_snap_s"), + "t_verify_s": stats.get("t_verify_s"), + "t_commit_s": stats.get("t_commit_s"), + "t_replay_s": stats.get("t_replay_s"), + "t_sync_s": stats.get("t_sync_s"), + "t_finalize_s": stats.get("t_finalize_s"), + "proj_no_replay_tps": proj, + "proj_no_replay_speedup": round(proj / max(base_tps, 1e-6), 2), + "baseline_disk_loads": base_miss, + "baseline_prefetch_loads": base_prefetch_loads, + "baseline_prefetch_hits": base_prefetch_hits, + "baseline_hit_rate": round(base_hit / max(base_hit + base_miss, 1), 3), + "spec_disk_loads": spec_miss, + "spec_prefetch_loads": spec_prefetch_loads, + "spec_prefetch_hits": spec_prefetch_hits, + "spec_hit_rate": round(spec_hit / max(spec_hit + spec_miss, 1), 3), + "disk_load_ratio": round(spec_miss / max(base_miss, 1), 2), + # 双源 acquire 分路计数:n_miss==0 走全 GPU 快路径;任一路由 miss 则整层落 host 慢路径 + # (.tolist 全批同步 + demand 读盘)。fallback 占比高 → 即使 hit 高,慢路径仍按"层"频繁触发。 + "gpu_fastpath": getattr(store._resident, "gpu_fastpath", None), + "gpu_fallback": getattr(store._resident, "gpu_fallback", None), + "pin_hot": PIN_HOT, + "pin_cal_tok": PIN_CAL_TOK, + "pinned_experts": store.pinned_count(), + "expert_slots": store.capacity, + "mlx_active_gb": round(after.mlx_active_bytes / 1e9, 2), + "mlx_peak_gb": round(after.mlx_peak_bytes / 1e9, 2), + "rss_gb": round(after.rss_bytes / 1e9, 2), + # 内存分块(清缓冲后的真实常驻):A 权重 / B 专家池 / C staging / D MTP / F 激活 + "mem_breakdown": _mem_breakdown(model, store, mtp), + "prefill_chunk": _cfg.prefill_chunk(), + "native_stage_cache": native_moe.stage_cache_stats(), + "bg_stats": (store._bg.stats() if getattr(store, "_bg", None) is not None else None), + "window_prof": _window_prof(), + "predict_recall": _predict_recall(), + "miss_attrib": _miss_attrib(), + "prefetch_tprof": _prefetch_tprof(stats.get("wall_s")), + "union_experts": _union_prof(), + } + # 噪声地板测量口径:DUMP_IDS=1 时把 baseline greedy 与 spec 的完整 token 序列打进日志, + # 供跨进程 run-to-run 逐位对比(默认关闭,不污染常规输出)。 + if os.environ.get("DUMP_IDS"): + print("DUMP_BASE_IDS " + json.dumps(list(base))) + print("DUMP_SPEC_IDS " + json.dumps(list(ids))) + # 字节真值校验自证口径:开 STG_VERIFY 时,把两处校验器的累计计数 + # (ok/bad/calls)打进日志。关键:让「0 BAD」可判真伪——若 calls==0 说明本配置根本 + # 没触发该校验器(如 STG_VERIFY 在 zerocopy_dual 路径不接线),此时 0 BAD 是空结论。 + if os.environ.get("STG_VERIFY"): + from mlx_streaming.core.cache import resident_pool as _rp_mod + from mlx_streaming.core.cache import virtual_pool as _vp_mod + _vsum = { + "STG_VERIFY.resident(verify_acquire_bytes)": dict(_rp_mod._stg_verify_state), + "STG_VERIFY.virtual(_verify_native_bytes)": dict(_vp_mod._stg_verify_state), + } + print("VERIFY_SUMMARY " + json.dumps(_vsum, ensure_ascii=False)) + print(json.dumps(result, ensure_ascii=False, indent=2)) + if _hprof: + # dump (gen, layer, t_fire) 原始日志,供离线分析回调触发时刻分布。 + _flat = _Nhp.staging_hprof_get() # 扁平 [gen,layer,t, ...] + _path = os.environ.get("STAGING_HPROF_OUT", "/tmp/ab/hprof.jsonl") + os.makedirs(os.path.dirname(_path), exist_ok=True) + _n = 0 + with open(_path, "w") as _f: + for i in range(0, len(_flat), 3): + _f.write(json.dumps([int(_flat[i]), int(_flat[i + 1]), float(_flat[i + 2])]) + "\n") + _n += 1 + print(f"[hprof] wrote {_n} handler records to {_path}") + + +def _tree_nbytes(obj): + from mlx.utils import tree_flatten + return sum(v.nbytes for _, v in tree_flatten(obj) if isinstance(v, mx.array)) + + +def _mem_breakdown(model, store, mtp): + """把 decode 稳态内存拆成各块(清掉 MLX 可回收缓冲后量真实常驻)。 + + A 常驻非专家权重 / B 专家常驻池 / C staging 预取 / D MTP drafter(扣共享 embedding)/ + F 激活+临时(= active底 − A−B−C−D)。返回 GiB 字典。 + """ + GIB = 1024 ** 3 + A = _tree_nbytes(model.parameters()) + rp = store._resident + B = sum(v.nbytes for pool in rp._pools.values() for v in pool.values()) + C = 0 + stg = getattr(store, "_staging", None) + if stg is not None: + from mlx.utils import tree_flatten + C = sum(v.nbytes for _, v in tree_flatten(getattr(stg, "__dict__", {})) + if isinstance(v, mx.array)) + # D:MTP drafter 全部权重减去与主模型共享的 embedding(run 里 mtp.embed_tokens = 主模型的) + D = _tree_nbytes(mtp.parameters()) + emb = getattr(getattr(mtp, "embed_tokens", None), "weight", None) + shared = 0 + if isinstance(emb, mx.array): + # 共享 embedding 在 A 已计入;若 mtp 的 embed 与主模型同一对象则从 D 扣除避免重复 + shared = _tree_nbytes(mtp.embed_tokens) if hasattr(mtp, "embed_tokens") else 0 + D = max(0, D - shared) + mx.clear_cache() + active = mx.get_active_memory() + peak = mx.get_peak_memory() + F = max(0, active - (A + B + C + D)) + pool_rows = sum(rp.allocated_slots(l) for l in rp._pools) + return { + "A_weights_gib": round(A / GIB, 3), + "B_expert_pool_gib": round(B / GIB, 3), + "B_pool_rows": pool_rows, + "C_staging_gib": round(C / GIB, 3), + "D_mtp_drafter_gib": round(D / GIB, 3), + "F_activation_temp_gib": round(F / GIB, 3), + "active_live_gib": round(active / GIB, 3), + "peak_gib": round(peak / GIB, 3), + } + + +def _union_prof(): + """按 seq 分桶汇报每层路由专家并集大小(avg/max/p99/分层);seq=K 桶即 MTP verify 的专家并集。 + + 需 UNION_PROF=1 才有数据。返回 {seq: {avg_union, max_union, p99_union, per_layer...}}; + verify 桶 = seq==K(verify_in=[x, d_1..d_{K-1}] 恰 K 个 token,见 mtp/generate.py)。 + U_max 即该桶各层并集的全局最大值,是池 cap 下限的正确性依据(cap 必须 ≥ U_max 才保证不溢出)。 + """ + from mlx_streaming.core.profiling import UNION_PROF as U, UNION_SAMPLES as S + if not U: + return None + + def _p(vals, q): + # 最近秩法分位数(vals 已升序):q∈[0,1]。样本少时直接给保守上界。 + if not vals: + return None + idx = max(0, min(len(vals) - 1, int(round(q * (len(vals) - 1))))) + return vals[idx] + + out = {} + for seq in sorted(U): + s, n = U[seq] + rec = {"avg_union": round(s / max(1, n), 2), "n_layer_calls": n} + # 由分层原始样本汇总出该 seq 桶的 U_max / p99（cap 下限的正确性依据）。 + layer_map = S.get(seq) + if layer_map: + flat = sorted(v for lst in layer_map.values() for v in lst) + rec["max_union"] = flat[-1] + rec["p99_union"] = _p(flat, 0.99) + rec["p50_union"] = _p(flat, 0.50) + rec["n_samples"] = len(flat) + # 分层分布：每层的 max / avg / 样本数（按层号排序）。 + per_layer = {} + for li in sorted(layer_map): + lst = layer_map[li] + per_layer[li] = { + "max": max(lst), + "avg": round(sum(lst) / len(lst), 2), + "n": len(lst), + } + rec["per_layer"] = per_layer + rec["max_layer_idx"] = max(per_layer, key=lambda li: per_layer[li]["max"]) + out[f"seq{seq}"] = rec + # 便捷:verify 桶 = seq==K(verify_in=[x, d_1..d_{K-1}] 恰 K 个 token) + verify_seq = K + verify = out.get(f"seq{verify_seq}") + return {"by_seq": out, "verify_seq": verify_seq, + "verify_avg_union": (verify or {}).get("avg_union"), + "verify_max_union": (verify or {}).get("max_union"), + "verify_p99_union": (verify or {}).get("p99_union")} + + +def _miss_attrib(): + from mlx_streaming.core.profiling import MISS_ATTRIB as M + if not M["n"]: + return None + routed = max(1, M["routed"]) + miss = M["miss_A_predicted"] + M["miss_B_unpredicted"] + out = { + "routed": M["routed"], + "hit_rate": round(M["resident_hit"] / routed, 4), + "miss_rate": round(miss / routed, 4), + # A：预测到却没进池（budget/时序/驱逐）；B：没预测到（召回缺口） + "miss_A_predicted": M["miss_A_predicted"], + "miss_B_unpredicted": M["miss_B_unpredicted"], + "A_share_of_miss": round(M["miss_A_predicted"] / max(1, miss), 4), + "B_share_of_miss": round(M["miss_B_unpredicted"] / max(1, miss), 4), + } + if M["dec_n"]: + # decode/verify 热路径专用（与上面 prefill-主导的全量分开看）： + dr = max(1, M["dec_routed"]) + dmiss = M["dec_miss_A"] + M["dec_miss_B"] + out["decode"] = { + "routed": M["dec_routed"], + "hit_rate": round(M["dec_resident_hit"] / dr, 4), + "miss_rate": round(dmiss / dr, 4), + "miss_A_predicted": M["dec_miss_A"], + "miss_B_unpredicted": M["dec_miss_B"], + "A_share_of_miss": round(M["dec_miss_A"] / max(1, dmiss), 4), + "B_share_of_miss": round(M["dec_miss_B"] / max(1, dmiss), 4), + # miss_A 细分:时序(pread 没完成) vs 驱逐(就绪过但 acquire 前不在池) + "miss_A_timing": M["dec_miss_A_timing"], + "miss_A_evicted": M["dec_miss_A_evicted"], + "A_timing_share": round(M["dec_miss_A_timing"] / max(1, M["dec_miss_A"]), 4), + "A_evicted_share": round(M["dec_miss_A_evicted"] / max(1, M["dec_miss_A"]), 4), + } + return out + + +def _predict_recall(): + from mlx_streaming.core.profiling import PREDICT_RECALL_PROF as P + if not P["n"]: + return None + return {"recall": round(P["hit"] / max(1, P["routed"]), 4), + "avg_routed": round(P["routed"] / P["n"], 2), "n": P["n"]} + + +def _window_prof(): + from mlx_streaming.core.profiling import WINDOW_PROF + if not WINDOW_PROF["n"]: + return None + return {"avg_ms": round(WINDOW_PROF["sum_s"] / WINDOW_PROF["n"] * 1000, 4), + "n": WINDOW_PROF["n"]} + + +def _prefetch_tprof(wall_s=None): + """预取 host 墙钟探针汇总(PREFETCH_TPROF=1 才有数据)。 + + 汇报每段:总秒数、占最终测量轮 wall 的百分比、单次调用均值(ms)。 + 注意:这是"主线程不可重叠的 host 时间";gate matmul / pool scatter 的 GPU 执行落在 + 前向末尾统一 eval,不在此口径,需用消融(NATIVE_NO_SUBMIT/NATIVE_NO_PROMOTE)差值测。 + """ + from mlx_streaming.core.profiling import PREFETCH_TPROF as T + if not (T["predict_n"] or T["promote_n"] or T["submit_n"]): + return None + + def _seg(s, n): + d = {"total_s": round(T[s], 4)} + if wall_s: + d["pct_wall"] = round(T[s] / wall_s * 100, 2) + if n: + d["avg_ms"] = round(T[s] / n * 1000, 4) + return d + + host_total = T["predict_s"] + T["submit_s"] + T["promote_s"] + out = { + "predict": _seg("predict_s", T["predict_n"]), + "submit": _seg("submit_s", T["submit_n"]), + "promote": _seg("promote_s", T["promote_n"]), + # promote 内部细分(take 锁读 / route 成员同步 / place 切片+scatter 入图) + "promote_take": _seg("take_s", T["promote_n"]), + "promote_route": _seg("route_s", T["promote_n"]), + "promote_place": _seg("place_s", T["promote_n"]), + "host_total_s": round(host_total, 4), + "host_total_pct_wall": (round(host_total / wall_s * 100, 2) if wall_s else None), + "counts": {"predict_n": T["predict_n"], "submit_n": T["submit_n"], + "promote_n": T["promote_n"], "place_experts": T["place_experts"]}, + } + return out + + +if __name__ == "__main__": + main() diff --git a/mlx_streaming/runtime/run_spec.py b/mlx_streaming/runtime/run_spec.py new file mode 100644 index 0000000..01abf79 --- /dev/null +++ b/mlx_streaming/runtime/run_spec.py @@ -0,0 +1,116 @@ +"""投机解码 + 流式专家 验证：target=流式 Qwen3-30B，draft=常驻 Qwen3-0.6B。 + +测「独立 draft 投机」在 I/O 流式 MoE 上到底有没有加速：对比不投机 vs 不同 +num_draft_tokens 的 decode tok/s 与草稿接受率。 + +环境变量： + MODEL/EXPERT_DIR/EXPERT_BITS/EXPERT_GROUP/EXPERT_SLOTS 同 run_streaming + DRAFT draft 模型路径（默认 /tmp/qwen3_draft） + NDRAFTS 要扫的 num_draft_tokens 列表（逗号分隔，默认 "2,3,4"） + MAXTOK/PROMPT +""" +import os +import time +import json + +import mlx.core as mx +from mlx_lm import load, stream_generate + +from mlx_streaming.core.mem import snapshot, reset_peak +from mlx_streaming.core.cache.expert_store import FileExpertStore +from mlx_streaming.core.prefetch.patch import patch_model_filebacked + +MODEL = os.environ.get("MODEL", "models/Qwen3-Next-80B-A3B-Instruct-MLX-8bit") +EXPERT_DIR = os.environ.get("EXPERT_DIR", "/tmp/mlx_qwen3_experts_2bit") +EXPERT_BITS = int(os.environ.get("EXPERT_BITS", "2")) +EXPERT_GROUP = int(os.environ.get("EXPERT_GROUP", "64")) +EXPERT_SLOTS = int(os.environ.get("EXPERT_SLOTS", "64")) +DRAFT = os.environ.get("DRAFT", "/tmp/qwen3_draft") +NDRAFTS = [int(x) for x in os.environ.get("NDRAFTS", "2,3,4").split(",")] +MAXTOK = int(os.environ.get("MAXTOK", "128")) +PROMPT = os.environ.get("PROMPT", "用三句话解释什么是混合专家模型。") + + +def _first_moe_dims(model): + for layer in model.layers: + mlp = getattr(layer, "mlp", None) + if mlp is not None and hasattr(mlp, "switch_mlp") and hasattr(mlp, "gate"): + gp = mlp.switch_mlp.gate_proj + return {"hidden": gp.input_dims, "moe_inter": gp.output_dims} + raise RuntimeError("无 MoE 层") + + +def _build_target(): + model, tok = load(MODEL, lazy=True) + dims = _first_moe_dims(model) + # 混合精度：从专家目录 meta 读 proj_bits/bits/group（有则优先于环境变量默认） + bits, group, proj_bits, layer_proj_bits = EXPERT_BITS, EXPERT_GROUP, None, None + meta_path = os.path.join(EXPERT_DIR, "_split_meta.json") + if os.path.exists(meta_path): + with open(meta_path) as f: + ed = json.load(f).get("dims", {}) + bits = ed.get("bits", bits) + group = ed.get("group_size", group) + proj_bits = ed.get("proj_bits") + if "per_layer_proj_bits" in ed: + layer_proj_bits = {int(k): v for k, v in ed["per_layer_proj_bits"].items()} + store = FileExpertStore(EXPERT_DIR, capacity=EXPERT_SLOTS) + patch_model_filebacked(model, store, dims["hidden"], dims["moe_inter"], + group, bits, proj_bits=proj_bits, + layer_proj_bits=layer_proj_bits) + return model, tok, store, {"bits": bits, "group": group, "proj_bits": proj_bits, + "layered": layer_proj_bits is not None} + + +def _run(model, tok, draft_model, num_draft, store): + store.reset_stats() if hasattr(store, "reset_stats") else None + n_tok = 0 + n_draft_acc = 0 + last_tps = 0.0 + kwargs = {} + if draft_model is not None: + kwargs["num_draft_tokens"] = num_draft + t0 = time.perf_counter() + for r in stream_generate(model, tok, prompt=PROMPT, max_tokens=MAXTOK, + draft_model=draft_model, **kwargs): + n_tok = r.generation_tokens + last_tps = r.generation_tps + if getattr(r, "from_draft", False): + n_draft_acc += 1 + wall = time.perf_counter() - t0 + return { + "num_draft_tokens": num_draft if draft_model is not None else None, + "gen_tokens": n_tok, + "tok_per_s": round(last_tps, 2), + "wall_s": round(wall, 2), + "draft_accepted_frac": round(n_draft_acc / n_tok, 3) if n_tok else 0.0, + "expert_hit_rate": round(store.hit_rate(), 3), + } + + +def main(): + reset_peak() + model, tok, store, qinfo = _build_target() + draft_model, _ = load(DRAFT) # 常驻小 draft + mx.eval(draft_model.parameters()) + + results = [] + # 1) 不投机基线 + results.append({"mode": "no_spec", **_run(model, tok, None, 0, store)}) + # 2) 投机：扫 num_draft_tokens + for nd in NDRAFTS: + results.append({"mode": "spec", **_run(model, tok, draft_model, nd, store)}) + + after = snapshot() + print(json.dumps({ + "target": MODEL, "draft": DRAFT, + "expert_bits": qinfo["bits"], "expert_proj_bits": qinfo["proj_bits"], + "expert_slots": EXPERT_SLOTS, + "rss_gb": round(after.rss_bytes / 1e9, 2), + "mlx_peak_gb": round(after.mlx_peak_bytes / 1e9, 2), + "results": results, + }, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/mlx_streaming/runtime/run_streaming.py b/mlx_streaming/runtime/run_streaming.py new file mode 100644 index 0000000..7e55d5b --- /dev/null +++ b/mlx_streaming/runtime/run_streaming.py @@ -0,0 +1,180 @@ +"""路线 B 端到端：lazy 加载 + 离线拆分专家 + 文件后端流式 + generate，量内存/速度/命中率。 + +环境变量： + MODEL 模型 repo 或本地路径（默认 models/Qwen3-Next-80B-A3B-Instruct-MLX-8bit） + EXPERT_DIR per-expert 拆分输出目录（默认 /tmp/mlx_qwen3_experts） + EXPERT_SLOTS 每层 LRU 专家槽数（默认 8=top_k），worst-case 常驻≈槽数×MoE层数 + PROMPT/MAXTOK 提示与生成 token 数 + WIRED_GB 可选，set_wired_limit 上限（GB） + CACHE_GB 可选，set_cache_limit 上限（GB），控制 MLX 缓冲复用上界 +""" +import os +import time +import json +import statistics + +import mlx.core as mx +from mlx_lm import load, generate + +from mlx_streaming import config +from mlx_streaming.core.mem import snapshot, reset_peak, clear_cache +from mlx_streaming.core.cache.expert_store import FileExpertStore +from mlx_streaming.prep.split_experts import split_model +from mlx_streaming.core.prefetch.patch import patch_model_filebacked +from mlx_streaming.model_builder import load_pool_profile + +MODEL = os.environ.get("MODEL", "models/Qwen3-Next-80B-A3B-Instruct-MLX-8bit") +EXPERT_DIR = os.environ.get("EXPERT_DIR", "/tmp/mlx_qwen3_experts") +EXPERT_SLOTS = int(os.environ.get("EXPERT_SLOTS", "8")) # 每层槽数 +PROMPT = os.environ.get("PROMPT", "用三句话解释什么是混合专家模型。") +MAXTOK = int(os.environ.get("MAXTOK", "128")) +# 稳态测速:warmup 跑满 MAXTOK(把整段会路由到的专家都装进常驻池 + 编译 Metal kernel), +# 再重复 REPEAT 次取中位数,避免冷启动/补池污染绝对 tok/s。WARMUP_TOK=0 关 warmup。 +WARMUP_TOK = int(os.environ.get("WARMUP_TOK", str(MAXTOK))) +REPEAT = int(os.environ.get("REPEAT", "3")) +WIRED_GB = os.environ.get("WIRED_GB") +CACHE_GB = os.environ.get("CACHE_GB") +CLEAR_ON_EVICT = os.environ.get("CLEAR_ON_EVICT", "0") == "1" +PIN_HOT = int(os.environ.get("PIN_HOT", "0")) # ③ 每层钉住的热专家数（0=关） +CAL_TOK = int(os.environ.get("CAL_TOK", "32")) # 校准生成 token 数 +# 专家量化格式覆盖：当 EXPERT_DIR 指向重量化目录(2/3-bit)时，文件后端 QSL 要用对应 bit/group +EXPERT_BITS = os.environ.get("EXPERT_BITS") +EXPERT_GROUP = os.environ.get("EXPERT_GROUP") + +def _first_moe_dims(model): + for layer in model.layers: + mlp = getattr(layer, "mlp", None) + if mlp is not None and hasattr(mlp, "switch_mlp") and hasattr(mlp, "gate"): + gp = mlp.switch_mlp.gate_proj + return { + "hidden": gp.input_dims, "moe_inter": gp.output_dims, + "group_size": getattr(gp, "group_size", 64), + "bits": getattr(gp, "bits", 4), + } + raise RuntimeError("模型里没有找到 MoE 层") + + +def main(): + if WIRED_GB: + fn = getattr(mx, "set_wired_limit", None) + if fn: + print("set_wired_limit 旧值=", fn(int(float(WIRED_GB) * 1e9))) + if CACHE_GB: + fn = getattr(mx, "set_cache_limit", None) + if fn: + print("set_cache_limit 旧值=", fn(int(float(CACHE_GB) * 1e9))) + + # 1. 拆分专家到磁盘（若尚未拆分） + if not os.path.exists(os.path.join(EXPERT_DIR, "_split_meta.json")): + print("拆分专家到", EXPERT_DIR, "...") + t = time.perf_counter() + meta = split_model(MODEL, EXPERT_DIR) + print("拆分完成", round(time.perf_counter() - t, 1), "s; MoE 层=", len(meta["moe_layers"])) + + reset_peak() + # 2. lazy 加载（不强制 eval 全部） + t0 = time.perf_counter() + model, tok = load(MODEL, lazy=True) + dims = _first_moe_dims(model) + # 专家目录 meta 是 bit/group/proj_bits 的权威来源（重量化目录都会写）；先用它覆盖， + # 再让 EXPERT_BITS/EXPERT_GROUP 环境变量做最高优先级手动覆盖。proj_bits 非空走混合精度。 + proj_bits = None + layer_proj_bits = None + meta_path = os.path.join(EXPERT_DIR, "_split_meta.json") + if os.path.exists(meta_path): + with open(meta_path) as f: + ed = json.load(f).get("dims", {}) + dims["bits"] = ed.get("bits", dims["bits"]) + dims["group_size"] = ed.get("group_size", dims["group_size"]) + proj_bits = ed.get("proj_bits") + if "per_layer_proj_bits" in ed: # 逐层混合：键转回 int 层号 + layer_proj_bits = {int(k): v for k, v in ed["per_layer_proj_bits"].items()} + if EXPERT_BITS: + dims["bits"] = int(EXPERT_BITS) + if EXPERT_GROUP: + dims["group_size"] = int(EXPERT_GROUP) + # 每层池预算 profile:默认从 {EXPERT_DIR}/pool_profile.json 自动启用(无损省内存)， + # EXPERT_POOL_PROFILE 可显式指定路径或设 none 关闭。命中率/输出/吞吐不变。 + layer_caps = load_pool_profile(EXPERT_DIR) + # 3. 文件后端 patch，丢弃常驻堆叠 switch_mlp + store = FileExpertStore(EXPERT_DIR, capacity=EXPERT_SLOTS, layer_caps=layer_caps, + clear_on_evict=CLEAR_ON_EVICT, record=PIN_HOT > 0) + n = patch_model_filebacked(model, store, dims["hidden"], dims["moe_inter"], + dims["group_size"], dims["bits"], + proj_bits=proj_bits, layer_proj_bits=layer_proj_bits) + t1 = time.perf_counter() + after_patch = snapshot() + + # 分块 prefill:把 mlx_lm.generate 内部 prefill 步长压到 config.prefill_chunk()(默认 2), + # 整段 prefill 的激活峰值 ∝prompt 长度 → ∝chunk,使 prefill 与 decode 同稳态。 + # PREFILL_CHUNK=0 时不传,回退 mlx_lm 默认 2048。 + _ps = config.prefill_chunk() + _gen_kw = {"prefill_step_size": _ps} if _ps > 0 else {} + + # 3.5 ③ 热专家常驻：先校准跑一遍统计激活频率，钉住每层最热的 PIN_HOT 个专家 + cal_s = 0.0 + if PIN_HOT > 0: + tc = time.perf_counter() + generate(model, tok, prompt=PROMPT, max_tokens=CAL_TOK, verbose=False, **_gen_kw) + for li in store.recorded_layers(): + store.pin(li, store.hot(li, PIN_HOT)) + store.record = False + store.reset_stats() + cal_s = round(time.perf_counter() - tc, 2) + + # 3.6 warmup:跑满 MAXTOK 把整段专家装进常驻池并编译 Metal kernel(PIN_HOT 已校准时 + # 这里仍补满全长,确保第一次正式测量即稳态)。 + if WARMUP_TOK > 0: + generate(model, tok, prompt=PROMPT, max_tokens=WARMUP_TOK, verbose=False, **_gen_kw) + + # 4. 生成:重复 REPEAT 次取中位数;最后一次清零专家统计供命中率口径对应稳态。 + reset_peak() + text = None + gen_runs = [] + for r in range(REPEAT): + if r == REPEAT - 1: + store.reset_stats() + tg = time.perf_counter() + text = generate(model, tok, prompt=PROMPT, max_tokens=MAXTOK, verbose=False, **_gen_kw) + gen_runs.append(round(MAXTOK / (time.perf_counter() - tg), 2)) + tok_per_s = statistics.median(gen_runs) + t2 = time.perf_counter() + clear_cache() + after_gen = snapshot() + + out = { + "mode": "streaming_filebacked", "model": MODEL, + "expert_dir": EXPERT_DIR, + "expert_bits": dims["bits"], "expert_group": dims["group_size"], + "proj_bits": proj_bits, "layered": layer_proj_bits is not None, + "per_layer_slots": EXPERT_SLOTS, "pool_profile": bool(layer_caps), + "wired_gb": WIRED_GB, "cache_gb": CACHE_GB, + "clear_on_evict": CLEAR_ON_EVICT, + "pin_hot": PIN_HOT, "cal_s": cal_s, + "warmup_tok": WARMUP_TOK, "repeat": REPEAT, + "patched_moe_layers": n, + "resident_experts": store.resident_count(), + "pinned_experts": store.pinned_count(), + "load_patch_s": round(t1 - t0, 2), + "tok_per_s": tok_per_s, + "tok_per_s_runs": gen_runs, + "tok_per_s_minmax": [min(gen_runs), max(gen_runs)], + "rss_gb_after_patch": round(after_patch.rss_bytes / 1e9, 2), + "rss_gb_after_gen": round(after_gen.rss_bytes / 1e9, 2), + "mlx_active_gb_after_gen": round(after_gen.mlx_active_bytes / 1e9, 2), + "mlx_peak_gb": round(after_gen.mlx_peak_bytes / 1e9, 2), + "expert_hit_rate": round(store.hit_rate(), 4), + "expert_hits": store.hits, "expert_misses": store.misses, + # GPU remap 取证:整层全命中走 GPU 快路径 vs 有 miss 回退 host 的层调用占比 + "gpu_fastpath": store._resident.gpu_fastpath, + "gpu_fallback": store._resident.gpu_fallback, + "gpu_fastpath_frac": round( + store._resident.gpu_fastpath + / max(1, store._resident.gpu_fastpath + store._resident.gpu_fallback), 4), + "sample": text[:240], + } + print(json.dumps(out, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/mlx_streaming/server/__init__.py b/mlx_streaming/server/__init__.py new file mode 100644 index 0000000..65b0769 --- /dev/null +++ b/mlx_streaming/server/__init__.py @@ -0,0 +1,9 @@ +"""OpenAI 兼容的 FastAPI server。 + +引擎复用 TUI 的 ChatBackend 抽象:SPARKLE_FAKE=1(或测试显式注入)时用 FakeBackend +免模型开发/测试;默认用 MLXBackend 装配真实流式 MoE 引擎。 +入口:python -m mlx_streaming.server +""" +from mlx_streaming.server.app import create_app + +__all__ = ["create_app"] diff --git a/mlx_streaming/server/__main__.py b/mlx_streaming/server/__main__.py new file mode 100644 index 0000000..2d03b9e --- /dev/null +++ b/mlx_streaming/server/__main__.py @@ -0,0 +1,95 @@ +"""sparkle OpenAI 兼容 server 入口:python -m mlx_streaming.server + +模型路径参数与 sparkle chat 一致(默认值同样来自 mlx_streaming/config.py 的环境变量)。 +SPARKLE_FAKE=1 时用 FakeBackend,免模型开发/测试。 +""" +import argparse +import os + +from mlx_streaming import config + + +def _build_parser(): + p = argparse.ArgumentParser( + prog="sparkle-server", + description="sparkle:OpenAI 兼容的流式 MoE 推理 server") + p.add_argument("--host", default="127.0.0.1", help="监听地址(默认 127.0.0.1)") + p.add_argument("--port", type=int, default=8317, help="监听端口(默认 8317)") + p.add_argument("--model", default=config.model_path(), help="主模型路径(MLX 量化)") + p.add_argument("--expert-dir", default=config.expert_dir(), + help="拆分后的 per-expert 目录") + p.add_argument("--mtp-out", default=config.mtp_out(), help="MTP 权重文件") + p.add_argument("--qn-config", default=config.qn_config(), + help="Qwen3-Next 配置 JSON") + p.add_argument("-k", "--k", type=int, default=3, help="MTP 投机宽度(默认 3)") + p.add_argument("-n", "--max-tokens", type=int, default=4096, + help="每轮最多生成的新 token 数(默认 4096)") + p.add_argument("--expert-slots", type=int, default=32, + help="常驻专家池容量(默认 32,同时作为侧区行数默认)") + p.add_argument("--spec-slots", type=int, default=None, + help="侧区行数 POOL_SPEC_SLOTS(默认跟随 --expert-slots)") + return p + + +def _safe_prefill_chunk(args) -> int: + """按专家池实际可安放容量算 prefill 步长上限，取 min(启动方给的值, 上限)。 + + 一次前向路由的唯一专家最多 chunk × top_k 个，真实区能安放的只有 + expert_slots − AUTOPIN 钉死数（钉死的永不驱逐）。超出会被 block.py 分流到 + host/fetch 路径：结果仍正确，但每层每 chunk 重读盘，prefill 慢几倍。 + 读不到 top_k 时按 Qwen3-Next 的 10 兜底（偏保守，宁可 chunk 小）。 + """ + import json + + top_k = 10 + try: + with open(args.qn_config, encoding="utf-8") as f: + top_k = max(1, int(json.load(f).get("num_experts_per_tok", top_k))) + except Exception: # noqa: BLE001 配置读不到/字段缺失都退到保守默认 + pass + slots = max(1, int(args.expert_slots)) + pinned = int(slots * config.autopin_budget_frac()) if config.autopin() else 0 + limit = max(1, (slots - pinned) // top_k) + want = int(os.environ.get("PREFILL_CHUNK") or limit) + if want > limit: + print(f"[PREFILL_CHUNK] {want} → {limit}：{slots} 槽扣掉 AUTOPIN 钉死 {pinned} 后只能安放 " + f"{slots - pinned} 个专家，chunk×top_k({top_k}) 不得超过它，否则 prefill 退到读盘慢路径。", + flush=True) + return min(want, limit) + + +def main(argv=None): + args = _build_parser().parse_args(argv) + + # KV_QUANT 维持 Sparkle 原始配方(开):此前"不按规则回答"已证实是 tools 未渲染所致, + # 与 KV 量化无关(且其通过质量门:token 一致率≥95%、logits cosine≥0.99)。 + # KV_QUANT=0 会使 prefill 慢 ~3 倍(fp16 KV 带宽),业务长提示下代价不可接受。 + # 显式 export KV_QUANT=0 可关闭做质量对照。 + + # 前缀快照头部的**上限**(实际长度由引擎按 system+tools 的真实边界自动取,见 + # MLXBackend._fixed_prefix_len)。写死数字曾是主要的慢因:业务 system+4 工具实测 + # 3972 token,固定切 2048 只覆盖一半,每个新会话都要白 prefill 1900+ token + # (prefill 约 30 tok/s,即多等一分钟)。这里只兜住极端情况下快照过大。 + os.environ.setdefault("PREFIX_SNAPSHOT_HEAD", "8192") + + # AUTOPIN 用一半槽位钉历史热专家(冷启动不慢热),另一半留给 demand 动态换入。 + os.environ.setdefault("AUTOPIN_BUDGET_FRAC", "0.5") + + # PREFILL_CHUNK 必须由槽位推导,不能让启动方拍数字:每个 chunk 的一次前向最多路由 + # chunk × top_k 个唯一专家,而真实区能安放的只有 expert_slots − AUTOPIN 钉死数。超了 + # 会被分流到 host/fetch 路径(正确但每层重读盘,慢几倍),槽位调小后写死的 chunk 就失效。 + os.environ["PREFILL_CHUNK"] = str(_safe_prefill_chunk(args)) + + import uvicorn + + from mlx_streaming.server.app import create_app + + # timeout_graceful_shutdown:SIGTERM 后最多等 3 秒就强制关闭连接。 + # 外部客户端的轮询持有 keep-alive 长连接,不设超时 uvicorn 会永远 + # 卡在 "Waiting for connections to close",进程不退、内存不释放。 + uvicorn.run(create_app(args), host=args.host, port=args.port, + timeout_graceful_shutdown=3) + + +if __name__ == "__main__": + main() diff --git a/mlx_streaming/server/admin.py b/mlx_streaming/server/admin.py new file mode 100644 index 0000000..52ecb8d --- /dev/null +++ b/mlx_streaming/server/admin.py @@ -0,0 +1,116 @@ +"""极简 admin 页:单文件 HTML + 内联 JS,不引框架/CDN。中文文案,2.5s 轮询 stats。""" + +ADMIN_HTML = """ + + + + +Sparkle 引擎控制台 + + + +