Preserve Unicode paths in releases and stabilize image manifests (#1476)

.github/workflows/website.yml

@@ -52,10 +52,13 @@ jobs:
           key: site-webp-v1-${{ runner.os }}-${{ hashFiles('website/requirements.txt', 'website/builder/site_assets.py', 'notes/pics/**', 'assets/**') }}
           restore-keys: site-webp-v1-${{ runner.os }}-
       - name: 生成并校验网站与配图
-        run: python website/refresh.py --from-checkout
+        run: |
+          python website/refresh.py --from-checkout
+          python website/deploy/package_site.py
       - name: 检查搜索和发布保护
         run: |
           python website/builder/test_site_assets.py
+          python website/deploy/test_package_site.py
           node website/builder/test_search.cjs
           npm test --prefix website/deploy
       - name: 发布网页和图片并校验正式域名

website/README.md

@@ -47,13 +47,16 @@ verify_updates.py 在临时副本中修改文章、替换配图、新增文章
 
 `.github/workflows/website.yml` 在 master 提交后、北京时间每天 09:17 和 21:17，以及手动 Run workflow 时运行。PR 仅验证合并后的源码，不读取部署凭据。CI 使用 `refresh.py --from-checkout` 从已检出的 Git 提交生成，源码中未提交的本机文件不会进入产物。
 
-通过图片、引用和搜索检查后，使用腾讯云官方 Makers SDK 发布整个 website/dist（包含网页和图片）到现有 cs-notes 项目，固定项目 ID 和中国站，上传前核对项目名称。清单与正式站一致时检查正式文件并跳过重复发布；有变化时等待腾讯云部署 Success，再核对正式域名的版本清单及 11 个关键页面/资源的实际 SHA-256，覆盖原图和优化配图。
+通过图片、引用和搜索检查后，使用 Python 标准库生成可重复的 UTF-8 发布包，再由腾讯云官方 Makers SDK 原样上传到现有 cs-notes 项目，包含整个 website/dist 的网页和图片。固定项目 ID 和中国站，上传前核对项目名称。清单与正式站一致时检查正式文件并跳过重复发布；有变化时等待腾讯云部署 Success，再核对正式域名的版本清单及 15 个关键页面/资源的实际 SHA-256，覆盖中文、空格文件名、图片清单、原图和优化配图。
+
+发布包由 `python3 website/deploy/package_site.py` 在构建后生成到忽略的 `.cache/site-release.zip`，仅包含已校验清单中的文件；打包阶段不传入令牌。SDK 的目录打包在当前平台解包后会将中文名转成 `#U…`，因此使用 SDK 的 archive 参数上传标准 ZIP，保留原有中文网址。
 
 本机检查发布保护（不需要令牌、不会上传）：
 
 ```bash
 npm ci --prefix website/deploy --ignore-scripts
 npm test --prefix website/deploy
+python3 website/deploy/test_package_site.py
 ```
 
 发布令牌只在仓库 Actions Secret `EDGEONE_API_TOKEN` 中配置，并仅传入可信 master 的发布步骤，不写入源码、网页、日志或报告。构建/校验失败时不会上传；发布后的域名检查失败会让任务失败，需查看部署记录后处理。删除令牌会停止后续发布，已经上线的网站不会被删除。

website/builder/site_assets.py

@@ -159,7 +159,7 @@ def optimize_images(dist, cache):
             return before + mapping.get(url, {}).get('path', url) + after
         parts = re.split(r'(```[^\n]*\n[\s\S]*?```|<!--[\s\S]*?-->)', path.read_text())
         path.write_text(''.join(part if i % 2 else image_pattern.sub(replace, part) for i, part in enumerate(parts)))
-    (dist / 'image-variants.json').write_text(json.dumps(mapping, ensure_ascii=False, indent=2) + '\n')
+    (dist / 'image-variants.json').write_text(json.dumps(mapping, ensure_ascii=False, indent=2, sort_keys=True) + '\n')
     return {'converted': len(mapping), 'originalBytes': sum(v['originalBytes'] for v in mapping.values()),
             'deliveryBytes': sum(v['deliveryBytes'] for v in mapping.values()), 'pixelIdentical': True}
 

website/builder/test_site_assets.py

@@ -1,11 +1,12 @@
 """Regression checks for generated search anchors and asset collection."""
 import sys
+import tempfile
 import unittest
 from collections import Counter
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
-from site_assets import heading_slug, referenced_paths, search_document
+from site_assets import heading_slug, optimize_images, referenced_paths, search_document
 
 
 class SearchIndexTests(unittest.TestCase):
@@ -26,6 +27,22 @@ class SearchIndexTests(unittest.TestCase):
         content = '<!-- <img src="/secret.png"> -->\n```html\n<img src="/example.png">\n```\n<img src="/notes/pics/actual.png">\n[正文](/notes/示例.md)\n'
         self.assertEqual(set(referenced_paths(content)), {'notes/pics/actual.png', 'notes/示例.md'})
 
+    def test_image_manifest_stable_across_file_creation_order(self):
+        from PIL import Image
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            outputs = []
+            for index, names in enumerate([['a.png', 'z.png'], ['z.png', 'a.png']]):
+                dist = root / str(index)
+                dist.mkdir()
+                for name in names:
+                    Image.new('RGB', (128, 128), 'red' if name == 'a.png' else 'blue').save(
+                        dist / name, compress_level=0)
+                report = optimize_images(dist, root / 'cache')
+                self.assertEqual(report['converted'], 2)
+                outputs.append((dist / 'image-variants.json').read_bytes())
+            self.assertEqual(outputs[0], outputs[1])
+
 
 if __name__ == '__main__':
     unittest.main()

website/deploy/deploy.mjs

@@ -23,7 +23,8 @@ export async function verifyPublished(manifestBytes, { fetcher = fetch, origin =
   const remote = await download(origin, 'asset-manifest.json', revision, fetcher);
   if (sha256(remote) !== revision) throw new Error('正式域名的版本清单尚未与构建一致。');
   const paths = ['index.html', 'build-info.json', 'README.md', 'site.css', 'site.js',
-    'search.js', 'search-paths.js', manifest.searchIndex, 'notes/MySQL.md'];
+    'search.js', 'search-paths.js', manifest.searchIndex, 'image-variants.json', 'notes/MySQL.md',
+    'notes/大模型应用.md', 'notes/Java 基础.md', 'notes/66. 构建乘积数组.md'];
   const records = new Map(manifest.files.map(file => [file.path, file]));
   for (const prefix of ['notes/pics/', 'optimized/']) {
     const image = manifest.files.find(file => file.path.startsWith(prefix));
@@ -42,7 +43,7 @@ export async function verifyPublished(manifestBytes, { fetcher = fetch, origin =
   return paths.length + 1;
 }
 
-export async function deploySite({ directory, getClient, fetcher = fetch,
+export async function deploySite({ directory, archive = resolve(directory, '../.cache/site-release.zip'), getClient, fetcher = fetch,
   sleep = ms => new Promise(done => setTimeout(done, ms)), attempts = 12 }) {
   const manifestBytes = await readFile(resolve(directory, 'asset-manifest.json'));
   const manifest = JSON.parse(manifestBytes);
@@ -59,7 +60,7 @@ export async function deploySite({ directory, getClient, fetcher = fetch,
       throw new Error('腾讯云项目标识与预期不一致，已停止发布。');
     }
     deployment = await makers.deployments.deploy({
-      projectId: PROJECT.projectId, artifact: { directory: resolve(directory) },
+      projectId: PROJECT.projectId, artifact: { archive: resolve(archive) },
       env: 'Production', wait: true, timeout: 600, pollInterval: 5,
       onStatusChange: ({ deployment }) => console.log(`腾讯云部署状态：${deployment.status}`),
     });

website/deploy/deploy.test.mjs

@@ -11,7 +11,8 @@ async function fixture(t) {
   t.after(() => rm(directory, { recursive: true, force: true }));
   const files = new Map(['index.html', 'build-info.json', 'README.md', 'site.css', 'site.js',
     'search.js', 'search-paths.js', 'search-index-example.json', 'notes/MySQL.md',
-    'notes/pics/example.png', 'optimized/example.webp']
+    'notes/pics/example.png', 'optimized/example.webp', 'image-variants.json',
+    'notes/大模型应用.md', 'notes/Java 基础.md', 'notes/66. 构建乘积数组.md']
     .map(path => [path, Buffer.from(`validated contents: ${path}`)]));
   const manifest = Buffer.from(JSON.stringify({ sourceCommit: 'source', generatorHash: 'generator',
     searchIndex: 'search-index-example.json', files: [...files].map(([path, data]) => ({
@@ -34,7 +35,7 @@ test('每日检查：清单一致时不请求凭据、不重复发布，同时
     },
   });
   assert.equal(report.result, 'unchanged');
-  assert.equal(report.verifiedFiles, 12);
+  assert.equal(report.verifiedFiles, 16);
   assert.ok(requests.includes('notes/MySQL.md'));
   assert.ok(requests.includes('notes/pics/example.png'));
   assert.ok(requests.includes('optimized/example.webp'));
@@ -57,7 +58,7 @@ test('新版本只向固定项目发布 dist，等候 Success 后重试正式域
         assert.equal(input.projectId, PROJECT.projectId);
         assert.equal(input.env, 'Production');
         assert.equal(input.wait, true);
-        assert.deepEqual(input.artifact, { directory: resolve(directory) });
+        assert.deepEqual(input.artifact, { archive: resolve(directory, '../.cache/site-release.zip') });
         published = true;
         return { status: 'Success', deploymentId: 'deployment-test' };
       } },
@@ -103,3 +104,12 @@ test('网页正常但优化图片缺失时不能宣告发布成功', async t =>
       ? '<html>SPA fallback</html>' : files.get(decodeURIComponent(url.pathname.slice(1)))),
   }), /文件校验不一致：optimized\/example.webp/);
 });
+
+test('中文或含空格文章返回首页时不能宣告发布成功', async t => {
+  const { directory, files } = await fixture(t);
+  await assert.rejects(deploySite({ directory, attempts: 1,
+    getClient: () => assert.fail('清单一致时不发布'),
+    fetcher: async url => new Response(url.pathname.includes(encodeURIComponent('大模型应用'))
+      ? '<html>SPA fallback</html>' : files.get(decodeURIComponent(url.pathname.slice(1)))),
+  }), /文件校验不一致：notes\/大模型应用.md/);
+});

website/deploy/package_site.py

@@ -0,0 +1,48 @@
+"""Create a reproducible UTF-8 ZIP from the validated manifest, without credentials."""
+import argparse
+import hashlib
+import json
+import tempfile
+import zipfile
+from pathlib import Path
+
+
+def create_archive(source, output):
+    source, output = Path(source).resolve(), Path(output).resolve()
+    if output.is_relative_to(source):
+        raise ValueError('发布包必须保存在网站产物目录外。')
+    manifest = json.loads((source / 'asset-manifest.json').read_text())
+    records = {item['path']: item for item in manifest['files']}
+    names = sorted([*records, 'asset-manifest.json'])
+    output.parent.mkdir(parents=True, exist_ok=True)
+    with tempfile.NamedTemporaryFile(dir=output.parent, suffix='.zip', delete=False) as temporary:
+        temporary_path = Path(temporary.name)
+    try:
+        with zipfile.ZipFile(temporary_path, 'w', compression=zipfile.ZIP_DEFLATED) as archive:
+            for name in names:
+                path = source / name
+                if not path.resolve().is_relative_to(source) or path.is_symlink():
+                    raise ValueError('非法发布路径：' + name)
+                data = path.read_bytes()
+                if name in records:
+                    record = records[name]
+                    if len(data) != record['bytes'] or hashlib.sha256(data).hexdigest() != record['sha256']:
+                        raise ValueError('打包前文件校验不一致：' + name)
+                entry = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0))
+                entry.compress_type = zipfile.ZIP_DEFLATED
+                entry.create_system = 3
+                entry.external_attr = 0o100644 << 16
+                archive.writestr(entry, data, compresslevel=6)
+        temporary_path.replace(output)
+    finally:
+        temporary_path.unlink(missing_ok=True)
+    return len(names)
+
+
+if __name__ == '__main__':
+    website = Path(__file__).resolve().parents[1]
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--source', type=Path, default=website / 'dist')
+    parser.add_argument('--output', type=Path, default=website / '.cache/site-release.zip')
+    args = parser.parse_args()
+    print('已校验并打包文件：' + str(create_archive(args.source, args.output)))

website/deploy/test_package_site.py

@@ -0,0 +1,54 @@
+"""Verify Unicode names, exact bytes and reproducibility of release archives."""
+import hashlib
+import json
+import tempfile
+import unittest
+import zipfile
+from pathlib import Path
+
+from package_site import create_archive
+
+
+class ArchiveTests(unittest.TestCase):
+    def fixture(self, root):
+        source = root / 'dist'
+        files = {'index.html': b'<h1>CS-Notes</h1>', 'notes/大模型应用.md': '# 中文文章'.encode(),
+                 'notes/Java 基础.md': b'Java', '.nojekyll': b''}
+        for name, data in files.items():
+            path = source / name
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_bytes(data)
+        manifest = {'files': [{'path': name, 'bytes': len(data),
+                    'sha256': hashlib.sha256(data).hexdigest()} for name, data in files.items()]}
+        (source / 'asset-manifest.json').write_text(json.dumps(manifest, ensure_ascii=False))
+        return source, files
+
+    def test_unicode_roundtrip_and_reproducible_bytes(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            source, files = self.fixture(root)
+            first, second = root / 'first.zip', root / 'second.zip'
+            self.assertEqual(create_archive(source, first), len(files) + 1)
+            create_archive(source, second)
+            self.assertEqual(first.read_bytes(), second.read_bytes())
+            with zipfile.ZipFile(first) as archive:
+                for name, data in files.items():
+                    self.assertEqual(archive.read(name), data)
+                    if not name.isascii():
+                        self.assertTrue(archive.getinfo(name).flag_bits & 0x800)
+
+    def test_modified_file_is_rejected_without_replacing_previous_archive(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            source, _ = self.fixture(root)
+            output = root / 'release.zip'
+            create_archive(source, output)
+            previous = output.read_bytes()
+            (source / 'notes/大模型应用.md').write_text('modified')
+            with self.assertRaisesRegex(ValueError, '校验不一致'):
+                create_archive(source, output)
+            self.assertEqual(output.read_bytes(), previous)
+
+
+if __name__ == '__main__':
+    unittest.main()