diff --git a/tests/v2/_differential_fixtures.py b/tests/v2/_differential_fixtures.py index 4fa45fbb..8ccb6025 100644 --- a/tests/v2/_differential_fixtures.py +++ b/tests/v2/_differential_fixtures.py @@ -48,8 +48,9 @@ def load_tool(stem: str) -> ModuleType: `tools/` is outside testpaths, and adding it would run --doctest-modules over the corpus builders, so these are imported - this way rather than made importable. None has import-time side - effects: each main() is behind a __name__ guard.""" + this way rather than made importable. Each main() is behind a + __name__ guard; the one import-time effect is that a corpus builder + appends its own directory to sys.path to find corpus_build.py.""" spec = importlib.util.spec_from_file_location( f"differential_{stem}", _TOOLS / f"{stem}.py") assert spec is not None and spec.loader is not None, stem diff --git a/tests/v2/test_corpus_build.py b/tests/v2/test_corpus_build.py new file mode 100644 index 00000000..84aa9423 --- /dev/null +++ b/tests/v2/test_corpus_build.py @@ -0,0 +1,114 @@ +"""Offline checks on tools/differential/corpus_build.py, the helpers the +labeled-corpus builders share. + +The builders' own tests reach these helpers through one builder at a +time; what is pinned here is each helper's contract, and that every +builder can find the module at all. tools/ is not a package, so modules +are loaded by path, as _differential_fixtures.py loads compare.py. +""" +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +from ._differential_fixtures import _TOOLS, load_tool + +corpus_build = load_tool("corpus_build") + +_BUILDERS = ("build_cldr_corpus", "build_gnd_corpus", "build_wikidata_corpus") + + +@pytest.mark.parametrize("stem", _BUILDERS) +def test_each_builder_finds_the_shared_module_on_its_own( + stem: str, tmp_path: Path) -> None: + # In a fresh interpreter, from an unrelated cwd and with safe-path on: + # in the test process an earlier builder's sys.path entry, or a cached + # import, would hide a builder that forgot its own. + loader = ("import importlib.util as u, sys; " + f"s = u.spec_from_file_location('b', {str(_TOOLS / (stem + '.py'))!r}); " + "m = u.module_from_spec(s); s.loader.exec_module(m); " + "print(m.corpus_lines is sys.modules['corpus_build'].corpus_lines)") + result = subprocess.run([sys.executable, "-P", "-c", loader], cwd=tmp_path, + capture_output=True, text=True, check=False) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "True" + + +def test_a_shape_must_be_an_int_or_none() -> None: + with pytest.raises(TypeError, match="shape must be an int or None, not '4'"): + corpus_build.corpus_lines([{"text": "A", "shape": "4"}]) + + +def test_an_untagged_line_sorts_before_the_tagged_ones_of_its_text() -> None: + # Shape 0 is the case to watch: a key that ties it with None leaves + # the order to set iteration, which a single text can pass by luck. + texts = [f"name {i}" for i in range(40)] + rows = [{"text": t, "shape": shape} for t in texts for shape in (0, None, 4)] + assert corpus_build.corpus_lines(rows) == [ + line for t in sorted(texts) + for line in (json.dumps(t), json.dumps({"name": t, "shape": 0}), + json.dumps({"name": t, "shape": 4}))] + + +def _row(text: str, order: str = "givenFirst") -> dict[str, object]: + return {"file": "Q1", "label": "x", "order": order, "text": text, + "shape": None, "fields": {"given": text}} + + +def test_write_corpus_writes_the_corpus_and_the_fields_file(tmp_path: Path) -> None: + corpus, fields = tmp_path / "corpus.jsonl", tmp_path / "fields.jsonl" + corpus_build.write_corpus([_row("B", "sorting"), _row("A")], corpus, fields) + assert corpus.read_text(encoding="utf-8") == '"A"\n"B"\n' + assert [json.loads(line)["order"] for line in + fields.read_text(encoding="utf-8").splitlines()] == ["givenFirst", "sorting"] + + +def test_a_refused_row_leaves_both_files_as_they_were(tmp_path: Path) -> None: + corpus, fields = tmp_path / "corpus.jsonl", tmp_path / "fields.jsonl" + corpus.write_text("old corpus\n", encoding="utf-8") + fields.write_text("old fields\n", encoding="utf-8") + with pytest.raises(ValueError, match="duplicate rendering Q1/x/givenFirst"): + corpus_build.write_corpus([_row("A"), _row("B")], corpus, fields) + assert corpus.read_text(encoding="utf-8") == "old corpus\n" + assert fields.read_text(encoding="utf-8") == "old fields\n" + + +def test_a_failed_write_leaves_both_files_as_they_were( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + corpus, fields = tmp_path / "corpus.jsonl", tmp_path / "fields.jsonl" + corpus.write_text("old corpus\n", encoding="utf-8") + fields.write_text("old fields\n", encoding="utf-8") + real = Path.write_text + + def full_disk(self: Path, text: str, encoding: str | None = None) -> int: + if self.name.startswith("fields"): + raise OSError("No space left on device") + return real(self, text, encoding=encoding) + monkeypatch.setattr(Path, "write_text", full_disk) + with pytest.raises(OSError, match="No space"): + corpus_build.write_corpus([_row("A")], corpus, fields) + monkeypatch.undo() + assert corpus.read_text(encoding="utf-8") == "old corpus\n" + assert fields.read_text(encoding="utf-8") == "old fields\n" + assert sorted(p.name for p in tmp_path.iterdir()) == ["corpus.jsonl", "fields.jsonl"] + + +def _check(body: bytes | str, source: str) -> dict[str, object]: + data = corpus_build.parse_json(body, source) + assert isinstance(data, dict) + return data + + +def test_an_interrupted_cache_write_leaves_no_cache_file( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + # The body goes to a .part file and is renamed into place; if the + # rename never happens, the next run must not find a cached answer. + def interrupted(src: object, dst: object) -> None: + raise OSError("interrupted") + monkeypatch.setattr(corpus_build.os, "replace", interrupted) + with pytest.raises(OSError, match="interrupted"): + corpus_build.cached_json(tmp_path, "k", "label", lambda: b'{"a": 1}', + _check, refresh=False) + assert not list(tmp_path.glob("*.json")) diff --git a/tests/v2/test_gnd_corpus.py b/tests/v2/test_gnd_corpus.py index 2ac06095..5b61d7b1 100644 --- a/tests/v2/test_gnd_corpus.py +++ b/tests/v2/test_gnd_corpus.py @@ -147,7 +147,8 @@ def test_a_record_its_fields_cannot_account_for_is_left_out( def test_the_corpus_holds_each_text_once() -> None: - rows = [{"text": "Björk"}, {"text": "Björk"}, {"text": "Abe, Kōbō"}] + rows = [{"text": "Björk", "shape": None}, {"text": "Björk", "shape": None}, + {"text": "Abe, Kōbō", "shape": None}] assert build.corpus_lines(rows) == ['"Abe, Kōbō"', '"Björk"'] diff --git a/tools/differential/README.md b/tools/differential/README.md index d87c3015..d4734862 100644 --- a/tools/differential/README.md +++ b/tools/differential/README.md @@ -403,9 +403,9 @@ A second view measures the locale packs (#672). `LOCALE_FILES` in `cldr_agreemen **The GND source.** `build_gnd_corpus.py` writes the GND's name parts in CLDR's field vocabulary (`forename` → `given`, `prefix` → `surname-prefix` with a one-word `surname` as `surname-core`, any other family → `surname` whole, `counting` → `generation`, and a `personalName` -- a person entered under the whole name, with no surname -- as `given` alone, so the family is expected empty), so `cldr_agreement.py` compares `gnd_fields.jsonl` by the same rules and records its disagreements in `gnd_agreement.json`, ratcheted by the same test. The locale view reads CLDR only. A composed rendering says so (`composed: true`); where a particle is glued to the surname (`d'Arenberg`, `al-Ǧaʿfarī`) the composed line states the glued word as a whole `surname`, since the comparison matches fields word by word, and the index form, where the particle is a word of its own, keeps the split. A family with no `prefix`, or with a surname of several words, is stated whole for the same kind of reason: GND does not split a particle inside a surname (`Vélez de Guevara`, `Zuylen van Nijevelt`), so what it leaves unsplit means "not stated", not "no particle", and only `family` is compared there. Raw lobid responses are cached outside the tree, in a `gnd/` directory under `$NAMEPARSER_TOOLS_CACHE` (default `~/.cache/nameparser-tools`); `--refresh` fetches them again. The build stops before writing anything when a stratum falls below its floor, a query matches nothing, or a pool pages short, so a lobid change fails it rather than shrinking the corpus; `--record` prints, per source, how many changes of each kind it accepts. -**Adding a source.** Map only what the source states. The comparison reads a missing field as a statement that the role is empty, so a field the source leaves unstated must be mapped so that its role is not compared at all: #688's first build wrote a prefixless GND surname as `surname-core`, and 19 renderings recorded the parser's `de` in `Vélez de Guevara` as a disagreement no fix could clear; its fix then missed the same gap inside a several-word core after a prefix (`Zuylen van Nijevelt, Julius van`), so test each way a field can be left unstated, not the one that was reported. A field must also occur in its rendering's text, word by word (`test_every_committed_field_occurs_in_its_rendering`), or it is expected empty too. A build must fail rather than shrink: give every stratum a floor, check a response's shape before caching it (an API error arrives as HTTP 200), and raise on a query that matches nothing. But tell a broken fetch from a fact about the data: #689's first fix raised on any item Wikidata reported missing, so one deleted name-part item anywhere in a pool would have stopped every rebuild, with `--refresh` advice that could not help; a missing part now only refuses the person stating it. +**Adding a source.** Map only what the source states. The comparison reads a missing field as a statement that the role is empty, so a field the source leaves unstated must be mapped so that its role is not compared at all: #688's first build wrote a prefixless GND surname as `surname-core`, and 19 renderings recorded the parser's `de` in `Vélez de Guevara` as a disagreement no fix could clear; its fix then missed the same gap inside a several-word core after a prefix (`Zuylen van Nijevelt, Julius van`), so test each way a field can be left unstated, not the one that was reported. A field must also occur in its rendering's text, word by word (`test_every_committed_field_occurs_in_its_rendering`), or it is expected empty too. A build must fail rather than shrink: give every stratum a floor, check a response's shape before caching it (an API error arrives as HTTP 200), and raise on a query that matches nothing. But tell a broken fetch from a fact about the data: #689's first fix raised on any item Wikidata reported missing, so one deleted name-part item anywhere in a pool would have stopped every rebuild, with `--refresh` advice that could not help; a missing part now only refuses the person stating it. The cache (`cache_dir`, `cached_json`, which checks a response before caching it and writes atomically) and the two output files (`corpus_lines`, `fields_lines`, `write_corpus`) are shared in `corpus_build.py`; a new builder imports them rather than copying another builder's. -**The Wikidata source.** `build_wikidata_corpus.py` exists for the two fields no other open source labels, and states them in `wikidata_fields.jsonl` as `surname2` (P1950, CLDR's own field name) and `patronymic` (P5056, which CLDR has no field for), beside `given` (P735) and `surname` (P734); its disagreements are in `wikidata_agreement.json`. A person's written names are token-aligned to the parts rather than composed from them: a text is kept only where every word is claimed by exactly one part, a connective (`bin`, `i`, `y`) joining the part after it, and a woman's Russian surname may match the feminine form of its item's label. ParsedName has no `surname2` or `patronymic` attribute, so `cldr_agreement.py` lists them in `PENDING_ROLES` and scores them inside `family` and `given_names` respectively -- a patronymic is not a family name, so `Vigdís Finnbogadóttir` expects the family empty -- and also scores each on its own the day ParsedName gains an attribute of that name (whether the merged role then drops it is for the 3.0 design). Until then the summary says how many renderings state each. CLDR's `surname2` is in the same position. A stratum that falls below its floor, a query that matches nothing, and a response that is not what was asked for each stop the build before anything is written; Malaysia's floor is set below its quota because only 17 Malaysian public figures with a P5056 statement matched the query on 2026-10-16, and 9 of them aligned. Raw responses are cached outside the tree (`$NAMEPARSER_TOOLS_CACHE` or `~/.cache/nameparser-tools/wikidata`); `--refresh` fetches them again. +**The Wikidata source.** `build_wikidata_corpus.py` exists for the two fields no other open source labels, and states them in `wikidata_fields.jsonl` as `surname2` (P1950, CLDR's own field name) and `patronymic` (P5056, which CLDR has no field for), beside `given` (P735) and `surname` (P734); its disagreements are in `wikidata_agreement.json`. A person's written names are token-aligned to the parts rather than composed from them: a text is kept only where every word is claimed by exactly one part, a connective (`bin`, `i`, `y`) joining the part after it, and a woman's Russian surname may match the feminine form of its item's label. ParsedName has no `surname2` or `patronymic` attribute, so `cldr_agreement.py` lists them in `PENDING_ROLES` and scores them inside `family` and `given_names` respectively -- a patronymic is not a family name, so `Vigdís Finnbogadóttir` expects the family empty -- and also scores each on its own the day ParsedName gains an attribute of that name (whether the merged role then drops it is for the 3.0 design). Until then the summary says how many renderings state each. CLDR's `surname2` is in the same position. A stratum that falls below its floor, a query that matches nothing, and a response that is not what was asked for each stop the build before anything is written; Malaysia's floor is set below its quota because only 17 Malaysian public figures with a P5056 statement matched the query on 2026-10-16, and 9 of them aligned. Raw responses are cached outside the tree, in a `wikidata/` directory under `$NAMEPARSER_TOOLS_CACHE` (default `~/.cache/nameparser-tools`); `--refresh` fetches them again. `--record` rewrites every record. `JA` needs the `ja` extra (`namedivider-python`), which only CI's `ja-extra` job installs: without it the `JA` ratchet skips, the summary says `not measured`, and `--record` keeps the recorded `ja` entries unchanged. Re-record with the extra installed (`uv sync --group dev --extra ja`) when a change moves Japanese. diff --git a/tools/differential/build_cldr_corpus.py b/tools/differential/build_cldr_corpus.py index 6d84a76f..bc2bd52e 100644 --- a/tools/differential/build_cldr_corpus.py +++ b/tools/differential/build_cldr_corpus.py @@ -48,9 +48,18 @@ import argparse import json +import sys import urllib.request from pathlib import Path +# The shared cache and file helpers (corpus_build.py), found by putting +# this directory on sys.path: see corpus_build's docstring for why. +_HERE = str(Path(__file__).resolve().parent) +if _HERE not in sys.path: + sys.path.append(_HERE) + +from corpus_build import corpus_lines, fields_lines, write_corpus # noqa: E402,F401 + HERE = Path(__file__).resolve().parent CLDR_TAG = "release-48" _REPO = "unicode-org/cldr" @@ -128,32 +137,6 @@ def parse_test_file(text: str, stem: str) -> list[dict[str, object]]: return rows -def corpus_lines(rows: list[dict[str, object]]) -> list[str]: - """One harness line per distinct (text, shape), sorted for stable diffs.""" - pairs: set[tuple[str, int | None]] = set() - for row in rows: - shape = row["shape"] - if shape is not None and not isinstance(shape, int): - raise TypeError(f"shape must be an int or None, not {shape!r}") - pairs.add((str(row["text"]), shape)) - return [json.dumps(text, ensure_ascii=False) if shape is None - else json.dumps({"name": text, "shape": shape}, ensure_ascii=False) - for text, shape in sorted(pairs, key=lambda p: (p[0], p[1] or 0))] - - -def fields_lines(rows: list[dict[str, object]]) -> list[str]: - """One line per rendering, sorted by its key. A key is unique per - rendering; a duplicate means the format was misread.""" - keyed: dict[str, dict[str, object]] = {} - for row in rows: - key = f"{row['file']}/{row['label']}/{row['order']}" - if key in keyed: - raise ValueError(f"duplicate rendering {key}") - keyed[key] = row - return [json.dumps(keyed[k], ensure_ascii=False, sort_keys=True) - for k in sorted(keyed)] - - def _get(url: str) -> bytes: request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) with urllib.request.urlopen(request, timeout=60) as response: @@ -170,8 +153,7 @@ def main() -> None: for name in names: text = _get(f"{RAW}/{_DIR}/{name}").decode("utf-8") rows.extend(parse_test_file(text, name[:-len(".txt")])) - CORPUS.write_text("\n".join(corpus_lines(rows)) + "\n", encoding="utf-8") - FIELDS.write_text("\n".join(fields_lines(rows)) + "\n", encoding="utf-8") + write_corpus(rows, CORPUS, FIELDS) LICENSE.write_bytes(_get(f"{RAW}/LICENSE")) print(f"CLDR {CLDR_TAG}: {len(names)} files, {len(rows)} renderings; " f"wrote {CORPUS.name}, {FIELDS.name}, {LICENSE.name}") diff --git a/tools/differential/build_gnd_corpus.py b/tools/differential/build_gnd_corpus.py index 2cfacd3d..f3c4e290 100644 --- a/tools/differential/build_gnd_corpus.py +++ b/tools/differential/build_gnd_corpus.py @@ -89,14 +89,22 @@ import argparse import hashlib -import json -import os +import sys import urllib.parse import urllib.request from collections.abc import Callable from pathlib import Path from typing import Any, NamedTuple +# The shared cache and file helpers (corpus_build.py), found by putting +# this directory on sys.path: see corpus_build's docstring for why. +_HERE = str(Path(__file__).resolve().parent) +if _HERE not in sys.path: + sys.path.append(_HERE) + +import corpus_build # noqa: E402 +from corpus_build import corpus_lines, fields_lines # noqa: E402,F401 + HERE = Path(__file__).resolve().parent CORPUS = HERE / "corpus_gnd.jsonl" FIELDS = HERE / "gnd_fields.jsonl" @@ -231,20 +239,9 @@ def floor(self) -> int: ) -def cache_dir() -> Path: - root = Path(os.environ.get("NAMEPARSER_TOOLS_CACHE", - Path.home() / ".cache" / "nameparser-tools")) - path = root / "gnd" - path.mkdir(parents=True, exist_ok=True) - return path - - def _checked(body: bytes | str, source: str) -> dict[str, Any]: """A lobid search response, or ValueError naming where it came from.""" - try: - data = json.loads(body) - except ValueError as error: - raise ValueError(f"{source} is not JSON ({error})") from None + data = corpus_build.parse_json(body, source) if not isinstance(data, dict) or "totalItems" not in data or "member" not in data: raise ValueError(f"{source} is not a lobid search response") return data @@ -252,18 +249,14 @@ def _checked(body: bytes | str, source: str) -> dict[str, Any]: def _get_json(params: dict[str, str | int], refresh: bool) -> dict[str, Any]: url = f"{SEARCH}?{urllib.parse.urlencode(params)}" - cached = cache_dir() / (hashlib.sha256(url.encode()).hexdigest()[:24] + ".json") - if cached.exists() and not refresh: - return _checked(cached.read_text(encoding="utf-8"), - f"{cached} (cached {url}; delete it or pass --refresh)") - request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) - with urllib.request.urlopen(request, timeout=120) as response: - body = response.read() - data = _checked(body, url) - partial = cached.with_suffix(".part") - partial.write_bytes(body) - os.replace(partial, cached) - return data + + def fetch() -> bytes: + request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(request, timeout=120) as response: + body: bytes = response.read() + return body + return corpus_build.cached_json(corpus_build.cache_dir("gnd"), url, url, + fetch, _checked, refresh) def _search(query: str, size: int, start: int, refresh: bool) -> dict[str, Any]: @@ -361,25 +354,6 @@ def row(order: str, text: str, fields: dict[str, str], composed: bool) -> dict[s return rows -def corpus_lines(rows: list[dict[str, Any]]) -> list[str]: - """One harness line per distinct text, sorted for stable diffs. No - GND rendering carries a shape: a comma or the default order reads it.""" - return [json.dumps(text, ensure_ascii=False) - for text in sorted({str(row["text"]) for row in rows})] - - -def fields_lines(rows: list[dict[str, Any]]) -> list[str]: - """One line per rendering, sorted by its key (GND id/stratum/order).""" - keyed: dict[str, dict[str, Any]] = {} - for row in rows: - key = f"{row['file']}/{row['label']}/{row['order']}" - if key in keyed: - raise ValueError(f"duplicate rendering {key}") - keyed[key] = row - return [json.dumps(keyed[k], ensure_ascii=False, sort_keys=True) - for k in sorted(keyed)] - - def build(refresh: bool) -> list[dict[str, Any]]: """Every stratum's rows, or ValueError naming each stratum that fell below its floor -- raised before main() writes anything.""" @@ -416,8 +390,7 @@ def main() -> None: ap.add_argument("--refresh", action="store_true", help="fetch from lobid again instead of reading the cache") rows = build(ap.parse_args().refresh) - CORPUS.write_text("\n".join(corpus_lines(rows)) + "\n", encoding="utf-8") - FIELDS.write_text("\n".join(fields_lines(rows)) + "\n", encoding="utf-8") + corpus_build.write_corpus(rows, CORPUS, FIELDS) LICENSE.write_text(LICENSE_TEXT, encoding="utf-8") print(f"GND: {len(rows)} renderings of {len({r['file'] for r in rows})} " f"names; wrote {CORPUS.name}, {FIELDS.name}, {LICENSE.name}") diff --git a/tools/differential/build_wikidata_corpus.py b/tools/differential/build_wikidata_corpus.py index 25b8f7d5..c0c9d7dd 100644 --- a/tools/differential/build_wikidata_corpus.py +++ b/tools/differential/build_wikidata_corpus.py @@ -22,9 +22,9 @@ compares it by the same rules. - wikidata_LICENSE -- the CC0 statement and the source credit. -Raw responses are cached outside the repository, in -$NAMEPARSER_TOOLS_CACHE or ~/.cache/nameparser-tools (as -tools/name_usage.py does), so a rebuild reads the same items until +Raw responses are cached outside the repository, in a wikidata/ +directory under $NAMEPARSER_TOOLS_CACHE (default +~/.cache/nameparser-tools, as tools/name_usage.py uses), so a rebuild reads the same items until --refresh fetches them again. Wikidata changes constantly; the QID on each fields line is the provenance. @@ -91,9 +91,7 @@ from __future__ import annotations import argparse -import hashlib -import json -import os +import sys import time import urllib.parse import urllib.request @@ -101,6 +99,15 @@ from pathlib import Path from typing import Any, NamedTuple +# The shared cache and file helpers (corpus_build.py), found by putting +# this directory on sys.path: see corpus_build's docstring for why. +_HERE = str(Path(__file__).resolve().parent) +if _HERE not in sys.path: + sys.path.append(_HERE) + +import corpus_build # noqa: E402 +from corpus_build import corpus_lines, fields_lines # noqa: E402,F401 + HERE = Path(__file__).resolve().parent CORPUS = HERE / "corpus_wikidata.jsonl" FIELDS = HERE / "wikidata_fields.jsonl" @@ -179,22 +186,11 @@ def floor(self) -> int: ) -def cache_dir() -> Path: - root = Path(os.environ.get("NAMEPARSER_TOOLS_CACHE", - Path.home() / ".cache" / "nameparser-tools")) - path = root / "wikidata" - path.mkdir(parents=True, exist_ok=True) - return path - - def _checked(body: bytes | str, source: str, kind: str) -> dict[str, Any]: """A WDQS result (`kind` "results") or a wbgetentities response ("entities"), or ValueError naming where it came from. An API error arrives with status 200, so the shape is what tells.""" - try: - data = json.loads(body) - except ValueError as error: - raise ValueError(f"{source} is not JSON ({error})") from None + data = corpus_build.parse_json(body, source) if not isinstance(data, dict) or kind not in data or "error" in data: detail = data.get("error") if isinstance(data, dict) else None raise ValueError(f"{source} has no {kind!r}" + (f": {detail}" if detail else "")) @@ -202,21 +198,16 @@ def _checked(body: bytes | str, source: str, kind: str) -> dict[str, Any]: def _fetch(url: str, data: bytes | None, refresh: bool, kind: str) -> dict[str, Any]: - key = url + "\n" + (data or b"").decode() - cached = cache_dir() / (hashlib.sha256(key.encode()).hexdigest()[:24] + ".json") - if cached.exists() and not refresh: - return _checked(cached.read_text(encoding="utf-8"), - f"{cached} (cached; delete it or pass --refresh)", kind) - headers = {"User-Agent": USER_AGENT, "Accept": "application/json"} - request = urllib.request.Request(url, data=data, headers=headers) - with urllib.request.urlopen(request, timeout=90) as response: - body = response.read() - parsed = _checked(body, url, kind) - partial = cached.with_suffix(".part") - partial.write_bytes(body) - os.replace(partial, cached) - time.sleep(1) # sequential, and gentle - return parsed + def fetch() -> bytes: + headers = {"User-Agent": USER_AGENT, "Accept": "application/json"} + request = urllib.request.Request(url, data=data, headers=headers) + with urllib.request.urlopen(request, timeout=90) as response: + body: bytes = response.read() + time.sleep(1) # sequential, and gentle + return body + return corpus_build.cached_json( + corpus_build.cache_dir("wikidata"), url + "\n" + (data or b"").decode(), url, + fetch, lambda body, source: _checked(body, source, kind), refresh) def query(stratum: Stratum) -> str: @@ -402,26 +393,6 @@ def renderings(qid: str, person: dict[str, Any], names: dict[str, Any], return rows -def corpus_lines(rows: list[dict[str, Any]]) -> list[str]: - """One harness line per distinct (text, shape), sorted for stable diffs.""" - pairs = {(str(row["text"]), row["shape"]) for row in rows} - return [json.dumps(text, ensure_ascii=False) if shape is None - else json.dumps({"name": text, "shape": shape}, ensure_ascii=False) - for text, shape in sorted(pairs, key=lambda p: (p[0], p[1] or 0))] - - -def fields_lines(rows: list[dict[str, Any]]) -> list[str]: - """One line per rendering, sorted by its key (QID/stratum/order).""" - keyed: dict[str, dict[str, Any]] = {} - for row in rows: - key = f"{row['file']}/{row['label']}/{row['order']}" - if key in keyed: - raise ValueError(f"duplicate rendering {key}") - keyed[key] = row - return [json.dumps(keyed[k], ensure_ascii=False, sort_keys=True) - for k in sorted(keyed)] - - def build(refresh: bool) -> list[dict[str, Any]]: """Every stratum's rows, or ValueError naming each stratum that fell below its floor -- raised before main() writes anything.""" @@ -467,8 +438,7 @@ def main() -> None: ap.add_argument("--refresh", action="store_true", help="fetch from Wikidata again instead of reading the cache") rows = build(ap.parse_args().refresh) - CORPUS.write_text("\n".join(corpus_lines(rows)) + "\n", encoding="utf-8") - FIELDS.write_text("\n".join(fields_lines(rows)) + "\n", encoding="utf-8") + corpus_build.write_corpus(rows, CORPUS, FIELDS) LICENSE.write_text(LICENSE_TEXT, encoding="utf-8") print(f"Wikidata: {len(rows)} renderings of {len({r['file'] for r in rows})} " f"names; wrote {CORPUS.name}, {FIELDS.name}, {LICENSE.name}") diff --git a/tools/differential/corpus_build.py b/tools/differential/corpus_build.py new file mode 100644 index 00000000..04ee55a7 --- /dev/null +++ b/tools/differential/corpus_build.py @@ -0,0 +1,115 @@ +"""What the labeled-corpus builders share: the cache, the files they write. + +build_cldr_corpus.py, build_gnd_corpus.py and build_wikidata_corpus.py +each write a corpus file and a fields file in the same formats, and GND +and Wikidata, the two that cache, keep their raw responses outside the +repository the same way. Not a builder itself: it writes nothing on +import, and has no main(). + +A builder imports it by name, after appending its own directory to +sys.path: that directory is usually sys.path[0] when a builder runs as a +script, but not under -P or PYTHONSAFEPATH (which AGENTS.md recommends +for comparisons) nor when the tests load a builder by path. It is +appended rather than inserted so that it cannot shadow a stdlib or +installed module; the cost is that an earlier `corpus_build` on sys.path +would shadow this one, and none exists. +""" +from __future__ import annotations + +import hashlib +import json +import os +from collections.abc import Callable, Iterable +from pathlib import Path +from typing import Any + + +def cache_dir(name: str) -> Path: + """The builder's cache directory, outside the repository: + $NAMEPARSER_TOOLS_CACHE or ~/.cache/nameparser-tools, as + tools/name_usage.py uses.""" + root = Path(os.environ.get("NAMEPARSER_TOOLS_CACHE", + Path.home() / ".cache" / "nameparser-tools")) + path = root / name + path.mkdir(parents=True, exist_ok=True) + return path + + +def parse_json(body: bytes | str, source: str) -> object: + """`body` parsed, or ValueError naming where it came from.""" + try: + return json.loads(body) + except ValueError as error: + raise ValueError(f"{source} is not JSON ({error})") from None + + +def cached_json(cache: Path, key: str, label: str, fetch: Callable[[], bytes], + check: Callable[[bytes | str, str], dict[str, Any]], + refresh: bool) -> dict[str, Any]: + """The response cached under `key`, or fetched and then cached. + + `check` parses a body and raises ValueError naming its source unless + it is the response asked for; it runs on a cached body too, so a + damaged file is named with the way out, and on a fetched one BEFORE + it is cached, so an error answer (often HTTP 200) never is. The write + is atomic: an interrupted run leaves no truncated cache file.""" + cached = cache / (hashlib.sha256(key.encode()).hexdigest()[:24] + ".json") + if cached.exists() and not refresh: + return check(cached.read_text(encoding="utf-8"), + f"{cached} (cached {label}; delete it or pass --refresh)") + body = fetch() + data = check(body, label) + partial = cached.with_suffix(".part") + partial.write_bytes(body) + os.replace(partial, cached) + return data + + +def corpus_lines(rows: Iterable[dict[str, Any]]) -> list[str]: + """One harness line per distinct (text, shape), sorted for stable + diffs: a bare JSON string, or {"name", "shape"} where a shape is set.""" + pairs: set[tuple[str, int | None]] = set() + for row in rows: + shape = row["shape"] + if shape is not None and not isinstance(shape, int): + raise TypeError(f"shape must be an int or None, not {shape!r}") + pairs.add((str(row["text"]), shape)) + # An untagged line sorts before every tagged one of the same text, so + # the order never rests on set iteration (shape 0 would tie with None). + return [json.dumps(text, ensure_ascii=False) if shape is None + else json.dumps({"name": text, "shape": shape}, ensure_ascii=False) + for text, shape in sorted(pairs, key=lambda p: (p[0], -1 if p[1] is None else p[1]))] + + +def fields_lines(rows: Iterable[dict[str, Any]]) -> list[str]: + """One line per rendering, sorted by its key (file/label/order, the + key cldr_agreement.py records under). A key is unique per rendering; + a duplicate means the source was misread.""" + keyed: dict[str, dict[str, Any]] = {} + for row in rows: + key = f"{row['file']}/{row['label']}/{row['order']}" + if key in keyed: + raise ValueError(f"duplicate rendering {key}") + keyed[key] = row + return [json.dumps(keyed[k], ensure_ascii=False, sort_keys=True) + for k in sorted(keyed)] + + +def write_corpus(rows: list[dict[str, Any]], corpus: Path, fields: Path) -> None: + """Write the corpus file and the fields file. Both are built, then + written to .part files, before either is renamed into place, so a + refused row (a duplicate rendering, a bad shape) or a failed write + leaves the committed pair as it was; only a crash between the two + renames could leave them out of step.""" + texts = {corpus: "\n".join(corpus_lines(rows)) + "\n", + fields: "\n".join(fields_lines(rows)) + "\n"} + parts = {path: path.with_name(path.name + ".part") for path in texts} + try: + for path, text in texts.items(): + parts[path].write_text(text, encoding="utf-8") + except BaseException: + for part in parts.values(): # beside the committed files: never leave one + part.unlink(missing_ok=True) + raise + for path, part in parts.items(): + os.replace(part, path)