diff --git a/website/builder/build.py b/website/builder/build.py index 3fa57e5b..28a24ee8 100644 --- a/website/builder/build.py +++ b/website/builder/build.py @@ -10,10 +10,12 @@ import posixpath import re import shutil import tempfile +import unicodedata import urllib.parse +from collections import Counter from pathlib import Path -from site_assets import copy_referenced_assets, optimize_images, write_manifest, write_search_index +from site_assets import copy_referenced_assets, heading_slug, optimize_images, write_manifest, write_search_index from check import check BASE = Path(__file__).resolve().parents[1] @@ -24,10 +26,13 @@ CACHE = BASE / '.cache/site-assets' VERSION_FILE = BASE / 'source-version.json' SOURCE_COMMIT = json.loads(VERSION_FILE.read_text())['commit'] if VERSION_FILE.is_file() else 'unknown' IMAGE_BY_NAME = {} +ANCHORS = {} SOURCE_IMAGE_HOST = 'cs-notes-1256109796.cos.ap-guangzhou.myqcloud.com' IMAGE = re.compile(r'(]*?\bsrc=["\'])([^"\']+)(["\'])', re.I) MD_IMAGE = re.compile(r'(!\[[^\]]*\]\()((?:[^()\s]|\([^()\n]*\))*)(\))') -MD_LINK = re.compile(r'(?`{3,}|~{3,})[^\n]*\n[\s\S]*?^(?P=fence)[`~]*[ \t]*$|', re.M) HTML_LINK = re.compile(r'(]*?\bhref=["\'])([^"\']+)(["\'])', re.I) @@ -58,6 +63,34 @@ def repository_images(): return result +def github_slug(text): + text = re.sub(r'<[^>]+>', '', text).strip().lower().replace('\\', '') + return ''.join(c for c in text if c in ' -_' or unicodedata.category(c)[0] in 'LN' + or unicodedata.category(c) == 'Mn').replace(' ', '-') + + +def anchor_map(path): + """Map GitHub heading anchors to the ids Docsify gives the same headings.""" + result, github, docsify, fence = {}, Counter(), Counter(), None + for line in re.sub(r'', '', path.read_text()).splitlines(): + marker = re.match(r'^\s*(`{3,}|~{3,})', line) + if marker: + fence = marker[1][0] if fence is None else (None if marker[1][0] == fence else fence) + continue + heading = re.match(r'^(#{1,6})\s+(.+?)(?:\s+#+)?\s*$', line) if fence is None else None + if heading: + slug = github_slug(heading[2]) + count = github[slug] + github[slug] += 1 + result.setdefault(slug + (f'-{count}' if count else ''), heading_slug(heading[2], docsify)) + return result + + +def site_anchor(path, fragment): + fragment = urllib.parse.unquote(fragment) + return ANCHORS.get(path, {}).get(fragment, fragment) + + def local_target(url, source, html_link=False): url = html.unescape(url) parsed = urllib.parse.urlsplit(url) @@ -75,6 +108,8 @@ def local_target(url, source, html_link=False): path = candidates[0].relative_to(ROOT).as_posix() elif parsed.netloc: return url + elif not path and url.startswith('#') and len(url) > 1: + return '#' + site_anchor(source.relative_to(ROOT).as_posix(), url[1:]) elif not path or url.startswith(('#', 'mailto:', 'data:', '//')): return url elif not path.startswith('/'): @@ -86,14 +121,14 @@ def local_target(url, source, html_link=False): if not html_link: result = '/' + urllib.parse.quote(path, safe='/') if parsed.fragment: - result += '#' + urllib.parse.quote(urllib.parse.unquote(parsed.fragment), safe='') + result += '#' + urllib.parse.quote(site_anchor(path, parsed.fragment), safe='') return result if path == 'README.md': route = '/' else: route = '/' + urllib.parse.quote(path[:-3], safe='/') if parsed.fragment: - route += '?id=' + urllib.parse.quote(urllib.parse.unquote(parsed.fragment), safe='') + route += '?id=' + urllib.parse.quote(site_anchor(path, parsed.fragment), safe='') return '/#' + route return '/' + urllib.parse.quote(path, safe='/') @@ -114,13 +149,17 @@ def render_markdown(path): part = MD_IMAGE.sub(image, part) part = MD_LINK.sub(lambda m: f'[{m[1]}]({local_target(m[2], path)})', part) return HTML_LINK.sub(lambda m: m[1] + local_target(m[2], path, html_link=True) + m[3], part) - parts = re.split(r'(```[^\n]*\n[\s\S]*?```|)', content) - return ''.join(part if i % 2 else convert(part) for i, part in enumerate(parts)) + result, last = [], 0 + for match in VERBATIM.finditer(content): + result += [convert(content[last:match.start()]), match[0]] + last = match.end() + return ''.join(result + [convert(content[last:])]) def build(optimize=True): - global IMAGE_BY_NAME + global IMAGE_BY_NAME, ANCHORS IMAGE_BY_NAME = repository_images() + ANCHORS = {p.relative_to(ROOT).as_posix(): anchor_map(p) for p in sources()} external = image_urls() replacements = {url: local_target(url, ROOT / 'README.md') for url in external} reused = {url: path for url, path in replacements.items() if path.startswith('/')} @@ -136,20 +175,20 @@ def build(optimize=True): target.parent.mkdir(parents=True, exist_ok=True) target.write_text(render_markdown(path)) homepage = (dist / 'README.md').read_text() - emojis = {'pencil2': '✏️', 'computer': '💻', 'cloud': '☁️', 'floppy_disk': '💾', + emojis = {'rocket': '🚀', 'pencil2': '✏️', 'computer': '💻', 'cloud': '☁️', 'floppy_disk': '💾', 'coffee': '☕', 'bulb': '💡', 'art': '🎨', 'wrench': '🔧', 'watermelon': '🍉', 'memo': '📝'} for name, symbol in emojis.items(): homepage = re.sub(r'^(## ):' + name + r':\s*', r'\1', homepage, flags=re.M) homepage = homepage.replace('#' + name + '-', '#') homepage = homepage.replace(':' + name + ':', symbol) - topics = [('算法', '✏️'), ('操作系统', '💻'), ('网络', '☁️'), ('面向对象', '🎨'), + topics = [('学习路线', '🚀'), ('算法', '✏️'), ('操作系统', '💻'), ('网络', '☁️'), ('面向对象', '🎨'), ('数据库', '💾'), ('Java', '☕'), ('系统设计', '💡'), ('工具', '🔧'), ('后记', '📝')] cards = '' - homepage = re.sub(r'^\| 算法[^\n]*\n\|[^\n]*\n\|[^\n]*', cards, homepage, count=1, flags=re.M) + homepage = re.sub(r'^\| (?:学习路线|算法)[^\n]*\n\|[^\n]*\n\|[^\n]*', cards, homepage, count=1, flags=re.M) homepage = homepage.replace('width="200px"', 'class="site-logo" width="200px"', 1) (dist / 'README.md').write_text('# CS-Notes\n\n' + homepage) note_paths = ['/' + str(p.relative_to(ROOT).with_suffix('')) for p in sorted((ROOT / 'notes').glob('*.md'))] @@ -161,7 +200,7 @@ def build(optimize=True): break sidebar.extend(['', '- **' + label + '**']) elif line.startswith('- ['): - sidebar.append(' ' + line) + sidebar.append(' ' + re.match(r'- \[[^\]]+\]\([^)]+\)', line).group(0)) (dist / '_sidebar.md').write_text('\n'.join(sidebar) + '\n') (dist / '_404.md').write_text('# 页面暂未找到\n\n请从目录或搜索查找笔记。\n\n[返回首页](/)\n') (dist / '.nojekyll').touch() diff --git a/website/builder/site_assets.py b/website/builder/site_assets.py index b68af64c..49e5a818 100644 --- a/website/builder/site_assets.py +++ b/website/builder/site_assets.py @@ -29,9 +29,30 @@ def plain_text(text): return re.sub(r'\s+', ' ', html.unescape(text)).strip() +def marked_inline(title): + # Docsify slugs the HTML that marked renders for the heading: backslash escapes are + # resolved, code spans and link text are kept, and text is HTML-escaped, so a quote + # becomes " and leaves "quot" in the slug. + def escape(text): + text = re.sub(r'&(?!#?\w+;)', '&', text) + return text.replace('<', '<').replace('>', '>').replace('"', '"').replace("'", ''') + out = [] + for part in re.split(r'(`+[^`]*`+|<\/?[A-Za-z][^>]*>)', title): + if part.startswith('`'): + out.append(escape(part.strip('`'))) + elif part.startswith('<'): + out.append(part) + else: + part = re.sub(r'!?\[([^\]]*)\]\([^)]*\)', r'\1', part) + part = re.sub(r'\\([!-/:-@\[-`{-~])', r'\1', part) + out.append(escape(part.replace('**', '').replace('__', ''))) + return ''.join(out) + + def heading_slug(title, counts): # Match the pinned Docsify 4.13.1 slugger, including numeric and duplicate headings. title = title.replace('{docsify-ignore}', '').replace('{docsify-ignore-all}', '') + title = marked_inline(title) slug = PUNCTUATION.sub('', re.sub(r'<[^>]+>', '', title.strip()).lower()) slug = re.sub(r'-+', '-', re.sub(r'\s', '-', slug)) slug = re.sub(r'^(\d)', r'_\1', slug) diff --git a/website/builder/test_site_assets.py b/website/builder/test_site_assets.py index 8a78f66a..04f747ae 100644 --- a/website/builder/test_site_assets.py +++ b/website/builder/test_site_assets.py @@ -16,6 +16,9 @@ class SearchIndexTests(unittest.TestCase): self.assertEqual(heading_slug('三、检索增强生成', counts), '三、检索增强生成') self.assertEqual(heading_slug('TCP 拥塞控制', counts), 'tcp-拥塞控制') self.assertEqual(heading_slug('TCP 拥塞控制', counts), 'tcp-拥塞控制-1') + # marked 先把引号转义成 "，Docsify 再去掉标点，留下 quot（Edge 渲染的 Java 基础页核对过） + self.assertEqual(heading_slug('new String("abc")', counts), 'new-stringquotabcquot') + self.assertEqual(heading_slug('1\\. 数据', counts), '_1-数据') def test_headings_in_code_do_not_change_real_anchors(self): doc = search_document(Path('示例.md'), '# 示例\n\n```python\n# TCP 拥塞控制\n```\n\n## TCP 拥塞控制\n慢开始\n')