Preserve Unicode paths in releases and stabilize image manifests (#1476)

website/deploy/deploy.mjs

@@ -23,7 +23,8 @@ export async function verifyPublished(manifestBytes, { fetcher = fetch, origin =
   const remote = await download(origin, 'asset-manifest.json', revision, fetcher);
   if (sha256(remote) !== revision) throw new Error('正式域名的版本清单尚未与构建一致。');
   const paths = ['index.html', 'build-info.json', 'README.md', 'site.css', 'site.js',
-    'search.js', 'search-paths.js', manifest.searchIndex, 'notes/MySQL.md'];
+    'search.js', 'search-paths.js', manifest.searchIndex, 'image-variants.json', 'notes/MySQL.md',
+    'notes/大模型应用.md', 'notes/Java 基础.md', 'notes/66. 构建乘积数组.md'];
   const records = new Map(manifest.files.map(file => [file.path, file]));
   for (const prefix of ['notes/pics/', 'optimized/']) {
     const image = manifest.files.find(file => file.path.startsWith(prefix));
@@ -42,7 +43,7 @@ export async function verifyPublished(manifestBytes, { fetcher = fetch, origin =
   return paths.length + 1;
 }
 
-export async function deploySite({ directory, getClient, fetcher = fetch,
+export async function deploySite({ directory, archive = resolve(directory, '../.cache/site-release.zip'), getClient, fetcher = fetch,
   sleep = ms => new Promise(done => setTimeout(done, ms)), attempts = 12 }) {
   const manifestBytes = await readFile(resolve(directory, 'asset-manifest.json'));
   const manifest = JSON.parse(manifestBytes);
@@ -59,7 +60,7 @@ export async function deploySite({ directory, getClient, fetcher = fetch,
       throw new Error('腾讯云项目标识与预期不一致，已停止发布。');
     }
     deployment = await makers.deployments.deploy({
-      projectId: PROJECT.projectId, artifact: { directory: resolve(directory) },
+      projectId: PROJECT.projectId, artifact: { archive: resolve(archive) },
       env: 'Production', wait: true, timeout: 600, pollInterval: 5,
       onStatusChange: ({ deployment }) => console.log(`腾讯云部署状态：${deployment.status}`),
     });

website/deploy/deploy.test.mjs

@@ -11,7 +11,8 @@ async function fixture(t) {
   t.after(() => rm(directory, { recursive: true, force: true }));
   const files = new Map(['index.html', 'build-info.json', 'README.md', 'site.css', 'site.js',
     'search.js', 'search-paths.js', 'search-index-example.json', 'notes/MySQL.md',
-    'notes/pics/example.png', 'optimized/example.webp']
+    'notes/pics/example.png', 'optimized/example.webp', 'image-variants.json',
+    'notes/大模型应用.md', 'notes/Java 基础.md', 'notes/66. 构建乘积数组.md']
     .map(path => [path, Buffer.from(`validated contents: ${path}`)]));
   const manifest = Buffer.from(JSON.stringify({ sourceCommit: 'source', generatorHash: 'generator',
     searchIndex: 'search-index-example.json', files: [...files].map(([path, data]) => ({
@@ -34,7 +35,7 @@ test('每日检查：清单一致时不请求凭据、不重复发布，同时
     },
   });
   assert.equal(report.result, 'unchanged');
-  assert.equal(report.verifiedFiles, 12);
+  assert.equal(report.verifiedFiles, 16);
   assert.ok(requests.includes('notes/MySQL.md'));
   assert.ok(requests.includes('notes/pics/example.png'));
   assert.ok(requests.includes('optimized/example.webp'));
@@ -57,7 +58,7 @@ test('新版本只向固定项目发布 dist，等候 Success 后重试正式域
         assert.equal(input.projectId, PROJECT.projectId);
         assert.equal(input.env, 'Production');
         assert.equal(input.wait, true);
-        assert.deepEqual(input.artifact, { directory: resolve(directory) });
+        assert.deepEqual(input.artifact, { archive: resolve(directory, '../.cache/site-release.zip') });
         published = true;
         return { status: 'Success', deploymentId: 'deployment-test' };
       } },
@@ -103,3 +104,12 @@ test('网页正常但优化图片缺失时不能宣告发布成功', async t =>
       ? '<html>SPA fallback</html>' : files.get(decodeURIComponent(url.pathname.slice(1)))),
   }), /文件校验不一致：optimized\/example.webp/);
 });
+
+test('中文或含空格文章返回首页时不能宣告发布成功', async t => {
+  const { directory, files } = await fixture(t);
+  await assert.rejects(deploySite({ directory, attempts: 1,
+    getClient: () => assert.fail('清单一致时不发布'),
+    fetcher: async url => new Response(url.pathname.includes(encodeURIComponent('大模型应用'))
+      ? '<html>SPA fallback</html>' : files.get(decodeURIComponent(url.pathname.slice(1)))),
+  }), /文件校验不一致：notes\/大模型应用.md/);
+});

website/deploy/package_site.py

@@ -0,0 +1,48 @@
+"""Create a reproducible UTF-8 ZIP from the validated manifest, without credentials."""
+import argparse
+import hashlib
+import json
+import tempfile
+import zipfile
+from pathlib import Path
+
+
+def create_archive(source, output):
+    source, output = Path(source).resolve(), Path(output).resolve()
+    if output.is_relative_to(source):
+        raise ValueError('发布包必须保存在网站产物目录外。')
+    manifest = json.loads((source / 'asset-manifest.json').read_text())
+    records = {item['path']: item for item in manifest['files']}
+    names = sorted([*records, 'asset-manifest.json'])
+    output.parent.mkdir(parents=True, exist_ok=True)
+    with tempfile.NamedTemporaryFile(dir=output.parent, suffix='.zip', delete=False) as temporary:
+        temporary_path = Path(temporary.name)
+    try:
+        with zipfile.ZipFile(temporary_path, 'w', compression=zipfile.ZIP_DEFLATED) as archive:
+            for name in names:
+                path = source / name
+                if not path.resolve().is_relative_to(source) or path.is_symlink():
+                    raise ValueError('非法发布路径：' + name)
+                data = path.read_bytes()
+                if name in records:
+                    record = records[name]
+                    if len(data) != record['bytes'] or hashlib.sha256(data).hexdigest() != record['sha256']:
+                        raise ValueError('打包前文件校验不一致：' + name)
+                entry = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0))
+                entry.compress_type = zipfile.ZIP_DEFLATED
+                entry.create_system = 3
+                entry.external_attr = 0o100644 << 16
+                archive.writestr(entry, data, compresslevel=6)
+        temporary_path.replace(output)
+    finally:
+        temporary_path.unlink(missing_ok=True)
+    return len(names)
+
+
+if __name__ == '__main__':
+    website = Path(__file__).resolve().parents[1]
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--source', type=Path, default=website / 'dist')
+    parser.add_argument('--output', type=Path, default=website / '.cache/site-release.zip')
+    args = parser.parse_args()
+    print('已校验并打包文件：' + str(create_archive(args.source, args.output)))

website/deploy/test_package_site.py

@@ -0,0 +1,54 @@
+"""Verify Unicode names, exact bytes and reproducibility of release archives."""
+import hashlib
+import json
+import tempfile
+import unittest
+import zipfile
+from pathlib import Path
+
+from package_site import create_archive
+
+
+class ArchiveTests(unittest.TestCase):
+    def fixture(self, root):
+        source = root / 'dist'
+        files = {'index.html': b'<h1>CS-Notes</h1>', 'notes/大模型应用.md': '# 中文文章'.encode(),
+                 'notes/Java 基础.md': b'Java', '.nojekyll': b''}
+        for name, data in files.items():
+            path = source / name
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_bytes(data)
+        manifest = {'files': [{'path': name, 'bytes': len(data),
+                    'sha256': hashlib.sha256(data).hexdigest()} for name, data in files.items()]}
+        (source / 'asset-manifest.json').write_text(json.dumps(manifest, ensure_ascii=False))
+        return source, files
+
+    def test_unicode_roundtrip_and_reproducible_bytes(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            source, files = self.fixture(root)
+            first, second = root / 'first.zip', root / 'second.zip'
+            self.assertEqual(create_archive(source, first), len(files) + 1)
+            create_archive(source, second)
+            self.assertEqual(first.read_bytes(), second.read_bytes())
+            with zipfile.ZipFile(first) as archive:
+                for name, data in files.items():
+                    self.assertEqual(archive.read(name), data)
+                    if not name.isascii():
+                        self.assertTrue(archive.getinfo(name).flag_bits & 0x800)
+
+    def test_modified_file_is_rejected_without_replacing_previous_archive(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            source, _ = self.fixture(root)
+            output = root / 'release.zip'
+            create_archive(source, output)
+            previous = output.read_bytes()
+            (source / 'notes/大模型应用.md').write_text('modified')
+            with self.assertRaisesRegex(ValueError, '校验不一致'):
+                create_archive(source, output)
+            self.assertEqual(output.read_bytes(), previous)
+
+
+if __name__ == '__main__':
+    unittest.main()