From 20296f3cbc339451ca43b52c4be20cac9b54e1bf Mon Sep 17 00:00:00 2001 From: Libin YANG Date: Sat, 12 Sep 2026 13:01:05 +0800 Subject: [PATCH] feat: generate edit URLs from a build-time source map Flattened pages now resolve GitHub edit links and committer paths from .edit_map.json instead of README frontmatter. Co-authored-by: Cursor --- .gitignore | 2 + build_site.py | 58 ++++++++++++-- hooks/committer.py | 38 ++++++---- hooks/edit_map.py | 55 ++++++++++++++ hooks/edit_url.py | 15 +++- tests/test_edit_map.py | 168 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 316 insertions(+), 20 deletions(-) create mode 100644 hooks/edit_map.py create mode 100644 tests/test_edit_map.py diff --git a/.gitignore b/.gitignore index f1db095b62c43..76462df79bde2 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,5 @@ docs/contest.md docs-en/lc/ docs-en/lcci/ docs-en/contest.md +docs/.edit_map.json +docs-en/.edit_map.json diff --git a/build_site.py b/build_site.py index 5a7f4c8606593..93fdb89314249 100644 --- a/build_site.py +++ b/build_site.py @@ -1,9 +1,14 @@ """Flatten problem READMEs and generate MkDocs nav grouped by number range.""" +import json import os +import re from collections import defaultdict from typing import Dict, List, NamedTuple, Optional, Set, Tuple +EDIT_MAP_NAME = ".edit_map.json" +_EDIT_URL_LINE = re.compile(r"^edit_url:.*\n?", re.MULTILINE) + class NavItem(NamedTuple): sort_key: Tuple[int, ...] @@ -163,9 +168,48 @@ def load_only_dirs() -> Optional[Set[str]]: return only -def collect_items() -> Tuple[Dict[str, List[NavItem]], Dict[str, List[NavItem]]]: +def posix_repo_path(path: str) -> str: + return path.replace("\\", "/") + + +def strip_frontmatter_edit_url(content: str) -> str: + if not content.startswith("---"): + return content + end = content.find("\n---", 3) + if end == -1: + return content + front = content[:end] + rest = content[end:] + stripped, n = _EDIT_URL_LINE.subn("", front, count=1) + if not n: + return content + return stripped + rest + + +def write_edit_maps(edit_maps: Dict[str, Dict[str, str]]) -> None: + for docs_root, mapping in edit_maps.items(): + os.makedirs(docs_root, exist_ok=True) + path = os.path.join(docs_root, EDIT_MAP_NAME) + with open(path, "w", encoding="utf-8") as f: + json.dump(mapping, f, ensure_ascii=False, indent=2) + f.write("\n") + + +def add_static_edit_mappings(edit_maps: Dict[str, Dict[str, str]]) -> None: + if os.path.isfile(os.path.join("docs", "contest.md")): + edit_maps.setdefault("docs", {})["contest.md"] = "solution/CONTEST_README.md" + if os.path.isfile(os.path.join("docs-en", "contest.md")): + edit_maps.setdefault("docs-en", {})["contest.md"] = ( + "solution/CONTEST_README_EN.md" + ) + + +def collect_items() -> Tuple[ + Dict[str, List[NavItem]], Dict[str, List[NavItem]], Dict[str, Dict[str, str]] +]: nav_cn: Dict[str, List[NavItem]] = defaultdict(list) nav_en: Dict[str, List[NavItem]] = defaultdict(list) + edit_maps: Dict[str, Dict[str, str]] = {"docs": {}, "docs-en": {}} only = load_only_dirs() for dir_name, (target_dir, depth) in dirs_mapping.items(): @@ -181,16 +225,18 @@ def collect_items() -> Tuple[Dict[str, List[NavItem]], Dict[str, List[NavItem]]] dest = f"{target_dir}/{num}.md" item = NavItem(parse_sort_key(num), num, name, dest) is_en = "README_EN" in path + docs_root = "docs-en" if is_en else "docs" (nav_en if is_en else nav_cn)[dir_name].append(item) - docs_dir = os.path.join("docs-en" if is_en else "docs", target_dir) + edit_maps[docs_root][dest] = posix_repo_path(path) + docs_dir = os.path.join(docs_root, target_dir) os.makedirs(docs_dir, exist_ok=True) with open(os.path.join(docs_dir, f"{num}.md"), "w", encoding="utf-8") as f: - f.write(content) + f.write(strip_frontmatter_edit_url(content)) nav_cn[dir_name].sort(key=lambda x: x.sort_key) nav_en[dir_name].sort(key=lambda x: x.sort_key) - return nav_cn, nav_en + return nav_cn, nav_en, edit_maps def replace_nav(config: str, nav: str) -> str: @@ -200,7 +246,9 @@ def replace_nav(config: str, nav: str) -> str: def main() -> None: - nav_cn, nav_en = collect_items() + nav_cn, nav_en, edit_maps = collect_items() + add_static_edit_mappings(edit_maps) + write_edit_maps(edit_maps) write_range_indexes(nav_cn["solution"], "docs", "lc", "zh") write_range_indexes(nav_en["solution"], "docs-en", "lc", "en") diff --git a/hooks/committer.py b/hooks/committer.py index c861b441d417f..37b49f81546ce 100644 --- a/hooks/committer.py +++ b/hooks/committer.py @@ -4,8 +4,9 @@ Optimizations: 1. on_pre_build: single git log pass builds {file -> last_mtime} index, replacing the original per-page subprocess spawn (3000+ pages -> 1 call). -2. on_files: scan all docs frontmatter to extract edit_url, - parallel-prefetch GitHub API for all cache misses before page processing. +2. on_files: resolve each page's main-branch path from .edit_map.json + (frontmatter edit_url is a fallback), then parallel-prefetch GitHub API + for all cache misses before page processing. 3. on_page_context: pure cache lookup, near-zero overhead. """ @@ -15,15 +16,22 @@ import re import random import subprocess +import sys import threading -import urllib.parse from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime, timezone from pathlib import Path from typing import Dict, List, Optional +from urllib.parse import quote import requests +_HOOKS_DIR = Path(__file__).resolve().parent +if str(_HOOKS_DIR) not in sys.path: + sys.path.insert(0, str(_HOOKS_DIR)) + +import edit_map # noqa: E402 + # --- Logging --- def _now() -> str: @@ -73,6 +81,16 @@ def _extract_edit_url(md_path: str) -> Optional[str]: return None +def _repo_path_for_file(src_path: str, docs_dir: str, md_path: str) -> Optional[str]: + repo_path = edit_map.repo_path_for(src_path, docs_dir) + if repo_path: + return repo_path + edit_url = _extract_edit_url(md_path) + if edit_url: + return edit_map.repo_path_from_edit_url(edit_url) + return None + + # --- Main plugin --- class CommitterPlugin: def __init__(self): @@ -113,10 +131,9 @@ def on_files(self, files, config): if not f.src_path.endswith(".md"): continue md_path = os.path.join(docs_dir, f.src_path) - edit_url = _extract_edit_url(md_path) - if not edit_url: + repo_path = _repo_path_for_file(f.src_path, docs_dir, md_path) + if not repo_path: continue - repo_path = self._repo_path_from_edit_url(edit_url) git_mtime = self._mtime_index.get(repo_path, 0) cached = self.page_authors.get(repo_path) cached_time = cached.get("retrieved") if cached else None @@ -165,7 +182,7 @@ def on_page_context(self, context, page, cfg, nav): if not page.edit_url or _exclude(page.file.src_path, []): return context - repo_path = self._repo_path_from_edit_url(page.edit_url) + repo_path = edit_map.repo_path_from_edit_url(page.edit_url) git_mtime = self._mtime_index.get(repo_path, 0) cached = self.page_authors.get(repo_path) cached_time = cached.get("retrieved") if cached else None @@ -263,14 +280,9 @@ def _fetch_authors_from_api(self, api_url: str) -> Optional[List[Dict]]: _log(f"Unexpected status {r.status_code}; retrying...", "WARN") return authors - @staticmethod - def _repo_path_from_edit_url(edit_url: str) -> str: - raw = edit_url.split("/edit/main/")[-1] - return urllib.parse.unquote(raw) - @staticmethod def _api_url_from_repo_path(repo_path: str) -> str: - quoted = urllib.parse.quote(repo_path) + quoted = quote(repo_path) return ( "https://api.github.com/repos/doocs/leetcode/commits" f"?path={quoted}&sha=main&per_page=100" diff --git a/hooks/edit_map.py b/hooks/edit_map.py new file mode 100644 index 0000000000000..e3c63f38a04b0 --- /dev/null +++ b/hooks/edit_map.py @@ -0,0 +1,55 @@ +"""Resolve flattened MkDocs pages back to their main-branch source paths.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Dict, Optional +from urllib.parse import quote, unquote + +EDIT_URL_PREFIX = "https://github.com/doocs/leetcode/edit/main/" +EDIT_MAP_NAME = ".edit_map.json" +_CACHE: Dict[str, Dict[str, str]] = {} + + +def reset_cache() -> None: + _CACHE.clear() + + +def load_edit_map(docs_dir: str) -> Dict[str, str]: + key = str(docs_dir) + cached = _CACHE.get(key) + if cached is not None: + return cached + path = Path(docs_dir) / EDIT_MAP_NAME + if not path.is_file(): + _CACHE[key] = {} + return _CACHE[key] + data = json.loads(path.read_text(encoding="utf-8")) + _CACHE[key] = {str(k).replace("\\", "/"): str(v) for k, v in data.items()} + return _CACHE[key] + + +def repo_path_for(src_path: str, docs_dir: str) -> Optional[str]: + dest = (src_path or "").replace("\\", "/") + return load_edit_map(docs_dir).get(dest) + + +def edit_url_for(repo_path: str) -> str: + return EDIT_URL_PREFIX + quote(repo_path, safe="/") + + +def repo_path_from_edit_url(edit_url: str) -> str: + return unquote(edit_url.split("/edit/main/")[-1]) + + +def resolve_edit_url( + src_path: str, docs_dir: str, meta: Optional[dict] = None +) -> Optional[str]: + repo_path = repo_path_for(src_path, docs_dir) + if repo_path: + return edit_url_for(repo_path) + page_edit_url = (meta or {}).get("edit_url") + if page_edit_url: + return str(page_edit_url) + return None diff --git a/hooks/edit_url.py b/hooks/edit_url.py index 21febda9cb00f..f6473c41f2a03 100644 --- a/hooks/edit_url.py +++ b/hooks/edit_url.py @@ -1,9 +1,20 @@ +import sys +from pathlib import Path + from mkdocs import plugins +_HOOKS_DIR = Path(__file__).resolve().parent +if str(_HOOKS_DIR) not in sys.path: + sys.path.insert(0, str(_HOOKS_DIR)) + +import edit_map # noqa: E402 + # https://www.mkdocs.org/dev-guide/plugins/#events @plugins.event_priority(100) def on_page_markdown(markdown, page, config, files): - page_edit_url = page.meta.get("edit_url") - page.edit_url = str(page_edit_url) if page_edit_url else None + src_path = getattr(getattr(page, "file", None), "src_path", "") or "" + docs_dir = config.get("docs_dir", "docs") + page.edit_url = edit_map.resolve_edit_url(src_path, docs_dir, page.meta) + return markdown diff --git a/tests/test_edit_map.py b/tests/test_edit_map.py new file mode 100644 index 0000000000000..3916a4307015c --- /dev/null +++ b/tests/test_edit_map.py @@ -0,0 +1,168 @@ +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / "hooks")) +import build_site # noqa: E402 +import committer # noqa: E402 +import edit_map # noqa: E402 +import edit_url # noqa: E402 + + +class StripEditUrlTest(unittest.TestCase): + def test_removes_edit_url_from_frontmatter(self): + src = ( + "---\n" + "comments: true\n" + "edit_url: https://github.com/doocs/leetcode/edit/main/x.md\n" + "tags:\n" + " - 数组\n" + "---\n\n" + "# [1. Two Sum](https://leetcode.cn/problems/two-sum)\n" + ) + out = build_site.strip_frontmatter_edit_url(src) + self.assertNotIn("edit_url:", out) + self.assertIn("comments: true", out) + self.assertIn("tags:", out) + self.assertIn("# [1. Two Sum]", out) + + def test_leaves_body_and_missing_field_alone(self): + src = "---\ncomments: true\n---\n\n# title\n" + self.assertEqual(build_site.strip_frontmatter_edit_url(src), src) + self.assertEqual(build_site.strip_frontmatter_edit_url("# no frontmatter\n"), "# no frontmatter\n") + + +class CollectItemsEditMapTest(unittest.TestCase): + def test_writes_map_and_strips_copied_frontmatter(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + problem = root / "solution" / "0000-0099" / "0001.Two Sum" + problem.mkdir(parents=True) + (problem / "README.md").write_text( + "---\ncomments: true\nedit_url: https://example.com/cn\n---\n\n" + "# [1. Two Sum](https://leetcode.cn/problems/two-sum)\n", + encoding="utf-8", + ) + (problem / "README_EN.md").write_text( + "---\ncomments: true\nedit_url: https://example.com/en\n---\n\n" + "# [1. Two Sum](https://leetcode.com/problems/two-sum)\n", + encoding="utf-8", + ) + cwd = os.getcwd() + try: + os.chdir(root) + _nav_cn, _nav_en, edit_maps = build_site.collect_items() + (root / "docs" / "contest.md").write_text("# contest\n", encoding="utf-8") + (root / "docs-en" / "contest.md").write_text("# contest\n", encoding="utf-8") + build_site.add_static_edit_mappings(edit_maps) + build_site.write_edit_maps(edit_maps) + finally: + os.chdir(cwd) + + zh = (root / "docs" / "lc" / "1.md").read_text(encoding="utf-8") + en = (root / "docs-en" / "lc" / "1.md").read_text(encoding="utf-8") + self.assertNotIn("edit_url:", zh) + self.assertNotIn("edit_url:", en) + self.assertEqual( + edit_maps["docs"]["lc/1.md"], + "solution/0000-0099/0001.Two Sum/README.md", + ) + self.assertEqual( + edit_maps["docs-en"]["lc/1.md"], + "solution/0000-0099/0001.Two Sum/README_EN.md", + ) + self.assertEqual( + edit_maps["docs"]["contest.md"], "solution/CONTEST_README.md" + ) + dumped = json.loads((root / "docs" / ".edit_map.json").read_text(encoding="utf-8")) + self.assertEqual(dumped["lc/1.md"], "solution/0000-0099/0001.Two Sum/README.md") + + +class EditUrlHookTest(unittest.TestCase): + def tearDown(self): + edit_map.reset_cache() + + def test_sets_edit_url_from_map(self): + with tempfile.TemporaryDirectory() as tmp: + docs = Path(tmp) / "docs" + docs.mkdir() + (docs / ".edit_map.json").write_text( + json.dumps({"lc/1.md": "solution/0000-0099/0001.Two Sum/README.md"}), + encoding="utf-8", + ) + edit_map.reset_cache() + page = SimpleNamespace( + file=SimpleNamespace(src_path="lc/1.md"), + meta={}, + edit_url=None, + ) + out = edit_url.on_page_markdown("# hi", page, {"docs_dir": str(docs)}, None) + self.assertEqual(out, "# hi") + self.assertEqual( + page.edit_url, + "https://github.com/doocs/leetcode/edit/main/solution/0000-0099/0001.Two%20Sum/README.md", + ) + + def test_falls_back_to_frontmatter(self): + with tempfile.TemporaryDirectory() as tmp: + docs = Path(tmp) / "docs" + docs.mkdir() + edit_map.reset_cache() + page = SimpleNamespace( + file=SimpleNamespace(src_path="index.md"), + meta={"edit_url": "https://github.com/doocs/leetcode/edit/main/README.md"}, + edit_url=None, + ) + edit_url.on_page_markdown("# hi", page, {"docs_dir": str(docs)}, None) + self.assertEqual( + page.edit_url, + "https://github.com/doocs/leetcode/edit/main/README.md", + ) + + def test_clears_edit_url_when_unmapped(self): + with tempfile.TemporaryDirectory() as tmp: + docs = Path(tmp) / "docs" + docs.mkdir() + edit_map.reset_cache() + page = SimpleNamespace( + file=SimpleNamespace(src_path="tags.md"), + meta={}, + edit_url="https://github.com/doocs/leetcode/edit/main/docs/tags.md", + ) + edit_url.on_page_markdown("# hi", page, {"docs_dir": str(docs)}, None) + self.assertIsNone(page.edit_url) + + +class CommitterLookupTest(unittest.TestCase): + def tearDown(self): + edit_map.reset_cache() + + def test_prefers_map_over_frontmatter(self): + with tempfile.TemporaryDirectory() as tmp: + docs = Path(tmp) / "docs" + docs.mkdir() + (docs / ".edit_map.json").write_text( + json.dumps({"lc/1.md": "solution/0000-0099/0001.Two Sum/README.md"}), + encoding="utf-8", + ) + md = docs / "lc" + md.mkdir() + (md / "1.md").write_text( + "---\nedit_url: https://github.com/doocs/leetcode/edit/main/stale.md\n---\n", + encoding="utf-8", + ) + edit_map.reset_cache() + self.assertEqual( + committer._repo_path_for_file("lc/1.md", str(docs), str(md / "1.md")), + "solution/0000-0099/0001.Two Sum/README.md", + ) + + +if __name__ == "__main__": + unittest.main()