diff --git a/tests/v2/test_cldr_agreement.py b/tests/v2/test_cldr_agreement.py index 5b43d457..0dd402ba 100644 --- a/tests/v2/test_cldr_agreement.py +++ b/tests/v2/test_cldr_agreement.py @@ -1,12 +1,13 @@ -"""The CLDR and GND agreement ratchets, and the comparison rules they rest on. +"""The agreement ratchets, and the comparison rules they rest on. tools/differential/cldr_agreement.py decides, for each rendering and each role, whether nameparser agrees with the source's stated fields -(CLDR's, and the GND's written in CLDR's vocabulary). The rules +(CLDR's, and the GND's, Wikidata's and the conventions table's written +in CLDR's vocabulary). The rules are pinned here first, each with the wrong answer the plausible alternative gives recorded beside it; the ratchet test at the end then holds each source's corpus to its recorded disagreements in -tools/differential/cldr_agreement.json and gnd_agreement.json. +tools/differential/, one record per source. tools/ is not a package, so the module is loaded by path, as _differential_fixtures.py loads compare.py. @@ -14,6 +15,7 @@ import importlib.util import json import sys +from collections import Counter from pathlib import Path from types import ModuleType @@ -127,6 +129,23 @@ class WithSurname2: assert agreement.got_roles(WithSurname2())["surname2"] == "García" # type: ignore[arg-type] +def test_an_honorific_is_scored_inside_the_suffix() -> None: + """A word of address written after the name (the conventions + source's `honorific`, #686) is a suffix, as nameparser reads さん.""" + roles = agreement.expected_roles({"given": "Murat", "honorific": "Bey"}, "Murat Bey") + assert roles["suffix"] == "Bey" + assert roles["title"] == "" + assert roles["given_names"] == "Murat" + assert roles["family"] == "" + + +def test_a_field_no_rule_reads_raises() -> None: + """A mistyped field would be expected in no role, and a row holding + it would pass as agreeing with an empty parse.""" + with pytest.raises(ValueError, match="honorfic"): + agreement.expected_roles({"given": "Murat", "honorfic": "Bey"}, "Murat Bey") + + def test_expected_roles_compare_particles_only_where_cldr_splits_them() -> None: plain = agreement.expected_roles({"given": "Iris", "surname": "Falke"}, "Iris Falke") assert set(plain) == {"title", "given_names", "family", "suffix"} @@ -220,7 +239,12 @@ def test_changes_reports_a_rendering_that_is_gone() -> None: #: exceed: a truncated file would otherwise pass both checks below, and #: an emptied one re-recorded would match its emptied record. _CORPORA = {"CLDR": ("corpus_cldr.jsonl", 1000), "GND": ("corpus_gnd.jsonl", 500), - "Wikidata": ("corpus_wikidata.jsonl", 500)} + "Wikidata": ("corpus_wikidata.jsonl", 500), + "Conventions": ("corpus_conventions.jsonl", 25)} +#: Sources whose fields file states only some of its corpus's names: the +#: conventions table leaves a row without fields where no field can state +#: it, and says why in the row's `unstated`. +_PARTIAL = frozenset({"Conventions"}) def test_every_source_has_a_corpus_file() -> None: @@ -228,7 +252,7 @@ def test_every_source_has_a_corpus_file() -> None: @pytest.mark.parametrize("source", sorted(_CORPORA)) -def test_the_corpus_and_the_fields_file_hold_the_same_renderings(source: str) -> None: +def test_the_fields_file_holds_the_corpus_renderings(source: str) -> None: fields, _ = agreement.sources()[source] corpus, least = _CORPORA[source] from_fields = {(row["text"], row["shape"]) for row in agreement.load_rows(fields)} @@ -238,7 +262,10 @@ def test_the_corpus_and_the_fields_file_hold_the_same_renderings(source: str) -> entry = json.loads(line) from_corpus.add((entry, None) if isinstance(entry, str) else (entry["name"], entry["shape"])) - assert from_corpus == from_fields + if source in _PARTIAL: + assert from_fields <= from_corpus + else: + assert from_corpus == from_fields assert len(from_fields) > least, f"the {source} corpus is missing or truncated" @@ -255,6 +282,27 @@ def test_every_committed_field_occurs_in_its_rendering(source: str) -> None: assert not missing +@pytest.mark.parametrize("source", sorted(set(_CORPORA) - {"CLDR"})) +def test_the_fields_partition_their_rendering(source: str) -> None: + """Every word of a rendering is stated by exactly one field, so a part + left out is a failure here rather than an expectation of empty. CLDR + is exempt for the reason above; the other sources' builders claim + every word, and the conventions table is written to.""" + fields, _ = agreement.sources()[source] + bad = [] + for row in agreement.load_rows(fields): + values = list(row["fields"].values()) + if agreement._has_space(row["text"]): + ok = (Counter(w for v in values for w in agreement._words(v)) + == Counter(agreement._words(row["text"]))) + else: + ok = (sum(len(agreement._squash(v)) for v in values) + == len(agreement._squash(row["text"]))) + if not ok: + bad.append((row["text"], row["fields"])) + assert not bad + + @pytest.mark.parametrize("source", sorted(_CORPORA)) def test_nameparser_still_agrees_with_each_source_where_it_did(source: str) -> None: fields, record = agreement.sources()[source] diff --git a/tests/v2/test_corpora.py b/tests/v2/test_corpora.py index b4472ae0..195d6af5 100644 --- a/tests/v2/test_corpora.py +++ b/tests/v2/test_corpora.py @@ -43,12 +43,14 @@ def test_shapes_corpus_matches_the_case_table() -> None: def test_conventions_corpus_matches_its_table() -> None: - """corpus_conventions.jsonl is generated from conventions.py's - hand-written table (#686).""" + """corpus_conventions.jsonl and conventions_fields.jsonl are + generated from conventions.py's hand-written table (#686).""" module = load_tool("build_conventions_corpus") - assert _checked_in("corpus_conventions.jsonl") == module.selected(), ( - "corpus_conventions.jsonl is stale: regenerate with " - "`uv run python tools/differential/build_conventions_corpus.py`") + for name, expected in (("corpus_conventions.jsonl", module.selected()), + ("conventions_fields.jsonl", module.fields_rows())): + assert _checked_in(name) == expected, ( + f"{name} is stale: regenerate with " + "`uv run python tools/differential/build_conventions_corpus.py`") def test_case_shape_ids_exist_in_the_inventory() -> None: diff --git a/tools/differential/README.md b/tools/differential/README.md index 40ec8f01..d8f17a3a 100644 --- a/tools/differential/README.md +++ b/tools/differential/README.md @@ -317,13 +317,11 @@ what a reader who knows the convention would take each part to be: uv run python tools/differential/build_conventions_corpus.py ``` -That expectation is a note, not a pin, and nothing scores against -it: many are stated in terms no field states (a father's or -grandfather's name, an honorific that is not a title, a married form), -some are undecided, and a fields file must state every part, since the -scorer reads an unstated one as an empty role (#686). A row whose -reading is settled can graduate to `tests/v2/cases.py`. The names are -invented, never copied from the references the conventions come +The note is prose. Where a field can state the parts, the row +states them too, and the builder writes those rows to +`conventions_fields.jsonl` for the agreement ratchet (below). A row +whose reading is settled can graduate to `tests/v2/cases.py`. The names +are invented, never copied from the references the conventions come from. ## What the corpora do not cover @@ -426,6 +424,8 @@ A second view measures the locale packs (#672). `LOCALE_FILES` in `cldr_agreemen **The Wikidata source.** `build_wikidata_corpus.py` exists for the two fields no other open source labels, and states them in `wikidata_fields.jsonl` as `surname2` (P1950, CLDR's own field name) and `patronymic` (P5056, which CLDR has no field for), beside `given` (P735) and `surname` (P734); its disagreements are in `wikidata_agreement.json`. A person's written names are token-aligned to the parts rather than composed from them: a text is kept only where every word is claimed by exactly one part, a connective (`bin`, `i`, `y`) joining the part after it, and a woman's Russian surname may match the feminine form of its item's label. ParsedName has no `surname2` or `patronymic` attribute, so `cldr_agreement.py` lists them in `PENDING_ROLES` and scores them inside `family` and `given_names` respectively -- a patronymic is not a family name, so `Vigdís Finnbogadóttir` expects the family empty -- and also scores each on its own the day ParsedName gains an attribute of that name (whether the merged role then drops it is for the 3.0 design). Until then the summary says how many renderings state each. CLDR's `surname2` is in the same position. A stratum that falls below its floor, a query that matches nothing, and a response that is not what was asked for each stop the build before anything is written; Malaysia's floor is set below its quota because only 17 Malaysian public figures with a P5056 statement matched the query on 2026-10-16, and 9 of them aligned. Raw responses are cached outside the tree, in a `wikidata/` directory under `$NAMEPARSER_TOOLS_CACHE` (default `~/.cache/nameparser-tools`); `--refresh` fetches them again. +**The conventions source.** `conventions_fields.jsonl` states the parts of the hand-written conventions corpus, from the `fields` of `conventions.py`'s rows, and its disagreements are in `conventions_agreement.json`. The names are invented, so the record is a worklist of conventions rather than a measure of real data: a patronymic read as a family name (`Hassan bin Yusof`), a title read as a given name (`U Kyaw Win`). It uses the Wikidata source's `patronymic`, written with its connective (`bin Yusof`), and adds `honorific`, a word of address written after the name (`Bey`, `ssi`, `san`), scored inside `suffix` as nameparser reads `さん` (Derek, 2026-10-16). A row no field can state honestly is left out of the fields file and stays in the corpus, which the snapshot still watches: the kunya `Abu Karim`, the `-né` married form `Szabóné Kiss Éva`, and `arap`/`ole`/`wa`, whose last word is a parent's name or a family name depending on the family. Such a row says why in its `unstated`, and the builder refuses a row with neither `fields` nor `unstated`, so this fields file states a subset of its corpus without a row's fields being lost unseen. Being hand-written, it is also held to two checks the generated sources pass anyway: its fields partition each text word for word (`test_the_fields_partition_their_rendering`), and the scorer raises on a field name no rule reads, for every source. + `--record` rewrites every record. `JA` needs the `ja` extra (`namedivider-python`), which only CI's `ja-extra` job installs: without it the `JA` ratchet skips, the summary says `not measured`, and `--record` keeps the recorded `ja` entries unchanged. Re-record with the extra installed (`uv sync --group dev --extra ja`) when a change moves Japanese. `_CORPUS_NAMES` in `tests/v2/_differential_fixtures.py` leaves the file out (2026-10-12): the tests reading it carry counts measured over the names the other corpora held. `corpus_gnd.jsonl` and `corpus_wikidata.jsonl` stay in it (2026-10-16): the counts those tests carry are floors and dated snapshots, which a larger corpus cannot falsify, and their invariants now walk both files' names too. A release figure counts GND's moved names in, its composed renderings included, since the names are real; AGENTS.md's rule of counting out names only `corpus_cldr.jsonl` holds stays CLDR's alone (Derek, 2026-10-16). Wikidata's moved names count in under the same rule, which singles out CLDR alone; they are real and written, not composed. `differential_corpus()` in `tests/v2/conftest.py`, which feeds the property tests, globs `corpus*.jsonl`, so it reads CLDR (since #652), GND and Wikidata, and the decomposed-twin test walks all their names. diff --git a/tools/differential/build_conventions_corpus.py b/tools/differential/build_conventions_corpus.py index 677e616b..b3fd7024 100644 --- a/tools/differential/build_conventions_corpus.py +++ b/tools/differential/build_conventions_corpus.py @@ -1,17 +1,20 @@ -"""Regenerate corpus_conventions.jsonl from conventions.py's table (#686). +"""Regenerate corpus_conventions.jsonl and conventions_fields.jsonl from +conventions.py's table (#686). The table holds invented names patterned on naming conventions the other corpora barely cover; conventions.py says where the conventions come from and why each row's expected reading is a note rather than a pin. A row read under a non-default order is written with the shapes.py id -for that order, so the snapshot reads it under that order. +for that order, so the snapshot reads it under that order. A row with +`fields` is also written to conventions_fields.jsonl, in the row format +cldr_agreement.py compares, keyed by the name and its order. Regenerate after editing the table: uv run python tools/differential/build_conventions_corpus.py -tests/v2/test_corpora.py pins the checked-in file against this module's -selection, so a row added without regenerating fails the suite. +tests/v2/test_corpora.py pins both checked-in files against this +module's selection, so a row added without regenerating fails the suite. """ from __future__ import annotations @@ -22,6 +25,7 @@ HERE = Path(__file__).resolve().parent OUT = HERE / "corpus_conventions.jsonl" +FIELDS_OUT = HERE / "conventions_fields.jsonl" def _load(stem: str) -> ModuleType: @@ -55,6 +59,9 @@ def selected() -> list[object]: seen: set[tuple[str, str | None]] = set() lines: list[object] = [] for row in CONVENTIONS: + if (row.fields is None) == (row.unstated is None): + raise ValueError(f"conventions.py row {row.name!r} needs exactly one " + f"of `fields` and `unstated` (why no field states it)") if (row.name, row.order) in seen: raise ValueError(f"conventions.py lists {row.name!r} under " f"{row.order or 'the default order'} twice") @@ -64,12 +71,26 @@ def selected() -> list[object]: return lines -def main() -> None: - lines = selected() - with OUT.open("w", encoding="utf-8") as fh: +def fields_rows() -> list[dict[str, object]]: + """The fields file's rows, for the table's rows that state fields.""" + selected() # the same row checks as the corpus + return [{"fields": row.fields, "file": "conventions", "label": row.name, + "order": row.order or "default", + "shape": None if row.order is None else _shape_for(row.order), + "text": row.name} + for row in CONVENTIONS if row.fields is not None] + + +def _write(path: Path, lines: list[object]) -> None: + with path.open("w", encoding="utf-8") as fh: for line in lines: - fh.write(json.dumps(line, ensure_ascii=False) + "\n") - print(f"wrote {len(lines)} names to {OUT.name}") + fh.write(json.dumps(line, ensure_ascii=False, sort_keys=True) + "\n") + print(f"wrote {len(lines)} lines to {path.name}") + + +def main() -> None: + _write(OUT, selected()) + _write(FIELDS_OUT, list(fields_rows())) if __name__ == "__main__": diff --git a/tools/differential/cldr_agreement.py b/tools/differential/cldr_agreement.py index c740ada7..6b04b45a 100644 --- a/tools/differential/cldr_agreement.py +++ b/tools/differential/cldr_agreement.py @@ -1,4 +1,4 @@ -"""Where nameparser disagrees with CLDR and the GND about a name's fields -- a ratchet. +"""Where nameparser disagrees with each source's stated fields -- a ratchet. CLDR's person-name test data (see build_cldr_corpus.py) states every name's fields separately. This module parses each CLDR rendering with the @@ -9,8 +9,11 @@ fields in CLDR's vocabulary, so it is a second SOURCE compared by the same rules, with its record in gnd_agreement.json, and so is the Wikidata corpus (build_wikidata_corpus.py, wikidata_agreement.json), -which states the two fields CLDR's vocabulary merges or lacks; "CLDR" -below means any source wherever the rule is about fields. For each source the +which states the two fields CLDR's vocabulary merges or lacks, and the +hand-written conventions table (build_conventions_corpus.py, +conventions_agreement.json), which adds `honorific`, a word of address +written after the name, scored inside `suffix`; "CLDR" below means any +source wherever the rule is about fields. For each source the test fails on: - a role that agreed now disagrees (a regression); @@ -24,8 +27,7 @@ uv run python tools/differential/cldr_agreement.py # summary uv run python tools/differential/cldr_agreement.py --record # re-record ---record rewrites every record, the GND one and the locale one below -included, and first prints, per source, how many of each kind of change +--record rewrites every record, each source's and the locale one below, and first prints, per source, how many of each kind of change it is accepting, so a re-record made for one source cannot slip another source's regressions in unseen. @@ -52,7 +54,14 @@ gains an attribute of that name; until then the summary says how many renderings state it. Scoring a field on its own does not end the merged scoring; only a 3.0 decision to take the field out of `family` or -`given_names` changes PENDING_ROLES. +`given_names` changes PENDING_ROLES. The conventions source's +`honorific` is not pending: it is scored inside `suffix` by design, as +nameparser reads `さん`, and ParsedName is not meant to gain it. + +A field name no rule reads raises: a mistyped `honorfic` would +otherwise be expected in no role and pass as agreement. KNOWN_FIELDS is +every name expected_roles reads, IGNORED_FIELDS what a source states +that no rendering writes (CLDR's `locale` and `given-informal`). Nothing is special-cased. A deliberately unnatural rendering, `さん` read as a suffix where CLDR calls it a title, a lone word under a family-first @@ -69,7 +78,8 @@ from pathlib import Path from typing import Any -from nameparser import FAMILY_FIRST, ParsedName, Parser, Policy, locales, parser_for +from nameparser import (FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, ParsedName, Parser, Policy, + locales, parser_for) ROLES = ("title", "given_names", "family", "suffix", "family_particles", "family_base") @@ -86,11 +96,17 @@ GND_RECORD = HERE / "gnd_agreement.json" WIKIDATA_FIELDS = HERE / "wikidata_fields.jsonl" WIKIDATA_RECORD = HERE / "wikidata_agreement.json" +CONVENTIONS_FIELDS = HERE / "conventions_fields.jsonl" +CONVENTIONS_RECORD = HERE / "conventions_agreement.json" RECORD_COMMAND = "uv run python tools/differential/cldr_agreement.py --record" #: shapes.py's FAMILY_FIRST shape, the only tag build_cldr_corpus.py writes. FAMILY_FIRST_SHAPE = 4 +#: shapes.py's FAMILY_FIRST_GIVEN_LAST shape, which a conventions row may name. +FAMILY_FIRST_GIVEN_LAST_SHAPE = 5 _PARSERS = {None: Parser(), - FAMILY_FIRST_SHAPE: Parser(policy=Policy(name_order=FAMILY_FIRST))} + FAMILY_FIRST_SHAPE: Parser(policy=Policy(name_order=FAMILY_FIRST)), + FAMILY_FIRST_GIVEN_LAST_SHAPE: + Parser(policy=Policy(name_order=FAMILY_FIRST_GIVEN_LAST))} #: The locale view: CLDR file -> the nameparser.locales pack its #: renderings are parsed with. One pack per file, so the locale record is @@ -152,12 +168,23 @@ def occurs(value: str, text: str) -> bool: return bool(_squash(value)) and _squash(value) in _squash(text) +#: Every field name expected_roles reads, and the ones it skips on purpose. +KNOWN_FIELDS = frozenset({"title", "given", "given2", "surname", "surname-prefix", + "surname-core", "generation", "credentials", "honorific", + *PENDING_ROLES}) +IGNORED_FIELDS = frozenset({"locale", "given-informal"}) + + def expected_roles(fields: dict[str, str], text: str) -> dict[str, str]: """What the source says each role holds IN THIS RENDERING. CLDR's formats drop fields by locale and order, so a role is built only from the fields whose value occurs in the text; a role with none expects ''. The particle and base views are compared only where the source states the split, i.e. the name has a surname-core.""" + unknown = set(fields) - KNOWN_FIELDS - IGNORED_FIELDS + if unknown: + raise ValueError(f"fields {sorted(unknown)} of {text!r} are read by no " + f"rule; use one of {sorted(KNOWN_FIELDS)}") def present(*names: str) -> str: # In the order the values occur in the text, not CLDR's field # order: an unspaced rendering compares in order, and one that @@ -173,7 +200,7 @@ def pending(role: str) -> tuple[str, ...]: roles = {"title": present("title"), "given_names": present("given", "given2", *pending("given_names")), "family": present(*family, *pending("family")), - "suffix": present("generation", "credentials")} + "suffix": present("generation", "credentials", "honorific")} if "surname-core" in fields: roles["family_particles"] = present("surname-prefix") roles["family_base"] = present("surname-core", *pending("family")) @@ -317,7 +344,8 @@ def sources() -> dict[str, tuple[Path, Path]]: """Each source of stated fields: name -> (fields file, record). Read at call time, so a test that points a record elsewhere is obeyed.""" return {"CLDR": (FIELDS, RECORD), "GND": (GND_FIELDS, GND_RECORD), - "Wikidata": (WIKIDATA_FIELDS, WIKIDATA_RECORD)} + "Wikidata": (WIKIDATA_FIELDS, WIKIDATA_RECORD), + "Conventions": (CONVENTIONS_FIELDS, CONVENTIONS_RECORD)} def _summary(name: str, rows: list[dict[str, Any]], diff --git a/tools/differential/conventions.py b/tools/differential/conventions.py index 71e4bf45..3e3b850c 100644 --- a/tools/differential/conventions.py +++ b/tools/differential/conventions.py @@ -12,9 +12,22 @@ `expect` is a NOTE, not a pin: what a reader who knows the convention would take each part to be, often in terms the model cannot express yet (a father's or grandfather's name, an honorific that is not a title, a -married form), and sometimes left undecided (`H. Budi Santoso`). Nothing -scores the parser against it (#686 decided against a fields file). Today's reading is deliberately not recorded here either: -snapshot.jsonl holds it, and a second copy would go stale. A row whose +married form). Nothing scores the parser against it. + +`fields` states the parts in CLDR's field vocabulary, as +cldr_agreement.py compares them, wherever a field can: `patronymic` is +scored inside the given names and `surname2` inside the family, as for +the Wikidata source, and `honorific` (a word of address written after +the name: Bey, ssi, san) inside the suffix. A row no field can state is +left without one rather than stated wrongly, since the comparison reads +an unstated part as an empty role, and says why in `unstated`; every +row carries `fields` or `unstated`, so neither can be left out by +accident. build_conventions_corpus.py writes conventions_fields.jsonl +from the `fields`. + +Today's reading is deliberately not recorded here: snapshot.jsonl holds +it, and conventions_agreement.json holds it where it disagrees with +`fields`. A second copy would go stale. A row whose reading is settled can graduate to tests/v2/cases.py, where it becomes a contract. @@ -31,127 +44,165 @@ class Convention(NamedTuple): name: str order: str | None expect: str + fields: dict[str, str] | None = None + unstated: str | None = None CONVENTIONS: tuple[Convention, ...] = ( Convention("Somali", "given + father's + grandfather's name, no family name", "Hodan Farah Warsame", None, "given Hodan; Farah and Warsame are her father's and grandfather's " - "given names, not a family name"), + "given names, not a family name", + fields={'given': 'Hodan', 'patronymic': 'Farah Warsame'}), Convention("Sikh", "religious name between given and family name", "Harpreet Kaur Dhillon", None, "given Harpreet; Kaur is the religious name Sikh women traditionally " - "take, written as a surname when no family name is; family Dhillon"), + "take, written as a surname when no family name is; family Dhillon", + fields={'given': 'Harpreet', 'given2': 'Kaur', 'surname': 'Dhillon'}), Convention("Tamil", "father's given name as a leading initial", "K. Sivakumar", None, "K. is the father's given name; Sivakumar is the person's own given " - "name; no family name"), + "name; no family name", + fields={'patronymic': 'K.', 'given': 'Sivakumar'}), Convention("Spanish", "H. (hijo) as the Jr. equivalent", "Pedro López Ruiz H.", None, "given Pedro; family López Ruiz (paternal and maternal surnames); " - "H. a generational suffix"), + "H. a generational suffix", + fields={'given': 'Pedro', 'surname': 'López', 'surname2': 'Ruiz', 'generation': 'H.'}), Convention("Spanish", "Vda. de (widow of) before the late husband's surname", "Carmen Díaz Vda. de Mora", None, "given Carmen; her own family name Díaz; Vda. de Mora marks her as " - "the widow of a Mora"), + "the widow of a Mora", + fields={'given': 'Carmen', 'surname': 'Díaz Vda. de Mora'}), Convention("Turkish", "oğlu ('son of') written apart, part of the family name", "Ali Kara Oğlu", None, - "given Ali; family Kara Oğlu, the separated spelling of Karaoğlu"), + "given Ali; family Kara Oğlu, the separated spelling of Karaoğlu", + fields={'given': 'Ali', 'surname': 'Kara Oğlu'}), Convention("Turkish", "honorific after the given name", "Murat Bey", None, - "given Murat; Bey an honorific, not a family name"), + "given Murat; Bey an honorific, not a family name", + fields={'given': 'Murat', 'honorific': 'Bey'}), Convention("Hungarian", "title after the name", "Szabó Péter úr", "FAMILY_FIRST", - "family Szabó, given Péter; úr ('Mr.') a title written last"), + "family Szabó, given Péter; úr ('Mr.') a word of address written last", + fields={'surname': 'Szabó', 'given': 'Péter', 'honorific': 'úr'}), Convention("Hungarian", "married form: -né on the husband's given name", "Szabó Péterné", "FAMILY_FIRST", "family Szabó; Péterné ('wife of Péter') stands where her given " - "name would"), + "name would", + fields={'surname': 'Szabó', 'given': 'Péterné'}), Convention("Korean", "two-syllable given name written as two words", "Kim Min Ho", "FAMILY_FIRST", - "family Kim; given Min Ho, one name in two words"), + "family Kim; given Min Ho, one name in two words", + fields={'surname': 'Kim', 'given': 'Min Ho'}), Convention("Vietnamese", "Thi as the female middle name", "Tran Thi Mai", "FAMILY_FIRST", "family Tran; middle Thi; given Mai, which the convention writes " "last (the order FAMILY_FIRST_GIVEN_LAST names; read under " "FAMILY_FIRST as the issue measured it); Mai is also a " - "suffix acronym, so either order reads it as a suffix"), + "suffix acronym, so either order reads it as a suffix", + fields={'surname': 'Tran', 'given2': 'Thi', 'given': 'Mai'}), Convention("Bulgarian", "patronymic between given and family name", "Ivan Petrov Dimitrov", None, "given Ivan; Petrov his patronymic, from his father Petar; family " - "Dimitrov"), + "Dimitrov", + fields={'given': 'Ivan', 'patronymic': 'Petrov', 'surname': 'Dimitrov'}), Convention("Filipino", "Ma. for María, maternal surname in the middle", "Ma. Teresa Cruz Bautista", None, "given Ma. Teresa (Ma. is María, not a title); Cruz the mother's " - "surname; family Bautista"), + "surname; family Bautista", + fields={'given': 'Ma. Teresa', 'surname2': 'Cruz', 'surname': 'Bautista'}), Convention("Korean", "romanized honorific, spaced", "Park Ji-hoon ssi", "FAMILY_FIRST", - "family Park; given Ji-hoon; ssi (씨) an honorific"), + "family Park; given Ji-hoon; ssi (씨) an honorific", + fields={'surname': 'Park', 'given': 'Ji-hoon', 'honorific': 'ssi'}), Convention("Japanese", "romanized honorific, spaced", "Suzuki san", "FAMILY_FIRST", - "family Suzuki; san an honorific; no given name"), + "family Suzuki; san an honorific; no given name", + fields={'surname': 'Suzuki', 'honorific': 'san'}), Convention("Japanese", "romanized honorific, hyphenated", "Suzuki-san", "FAMILY_FIRST", - "family Suzuki; -san an honorific glued to it"), + "family Suzuki; -san an honorific glued to it", + fields={'surname': 'Suzuki', 'honorific': '-san'}), Convention("Burmese", "no family name; honorific U for a man", "U Kyaw Win", None, - "U an honorific; Kyaw Win the whole personal name; no family name"), + "U an honorific; Kyaw Win the whole personal name; no family name", + fields={'title': 'U', 'given': 'Kyaw Win'}), Convention("Burmese", "honorific Daw for a woman", "Daw Thida", None, - "Daw an honorific; Thida the whole personal name"), + "Daw an honorific; Thida the whole personal name", + fields={'title': 'Daw', 'given': 'Thida'}), Convention("Thai", "Khun before the given name", "Khun Malee", None, - "Khun a polite title; Malee the given name, which Thai address uses"), + "Khun a polite title; Malee the given name, which Thai address uses", + fields={'title': 'Khun', 'given': 'Malee'}), Convention("Malay", "bin ('son of') patronymic, no family name", "Hassan bin Yusof", None, - "given Hassan; bin Yusof his patronymic; no family name"), + "given Hassan; bin Yusof his patronymic; no family name", + fields={'given': 'Hassan', 'patronymic': 'bin Yusof'}), Convention("Malay", "bin abbreviated B.", "Hassan B. Yusof", None, - "as Hassan bin Yusof: B. is bin, not a middle initial"), + "as Hassan bin Yusof: B. is bin, not a middle initial", + fields={'given': 'Hassan', 'patronymic': 'B. Yusof'}), Convention("Malaysian Indian", "a/l (anak lelaki, 'son of')", "Suresh a/l Muthu", None, - "given Suresh; a/l Muthu his patronymic; no family name"), + "given Suresh; a/l Muthu his patronymic; no family name", + fields={'given': 'Suresh', 'patronymic': 'a/l Muthu'}), Convention("Malaysian Indian", "a/p (anak perempuan, 'daughter of')", "Kavitha a/p Raman", None, - "given Kavitha; a/p Raman her patronymic; no family name"), + "given Kavitha; a/p Raman her patronymic; no family name", + fields={'given': 'Kavitha', 'patronymic': 'a/p Raman'}), Convention("Indonesian", "H. (Haji), a title shaped like an initial", "H. Budi Santoso", None, - "H. a title (one who has made the hajj); given Budi; Santoso a " - "second given name or a family name"), + "H. a title (one who has made the hajj); given Budi Santoso, " + "the whole personal name: Indonesian law recognizes no surname", + fields={'title': 'H.', 'given': 'Budi Santoso'}), Convention("Kalenjin", "arap ('son of')", "David arap Kirui", None, - "given David; arap Kirui, used as a family name"), + "given David; arap Kirui, used as a family name", + unstated="arap Kirui is a patronymic or a family name by family"), Convention("Maasai", "ole ('son of')", "Joseph ole Sankale", None, - "given Joseph; ole Sankale, used as a family name"), + "given Joseph; ole Sankale, used as a family name", + unstated="ole Sankale is a patronymic or a family name by family"), Convention("Kikuyu", "wa ('of')", "Wanjiru wa Kamau", None, - "given Wanjiru; wa Kamau names her father"), + "given Wanjiru; wa Kamau names a parent, her father or her mother", + unstated="wa Kamau is a patronymic, a matronymic or a family name " + "by family"), Convention("Hong Kong", "English given name before the Chinese name", "Michael Wong Kar Ming", None, - "given Michael; family Wong; Kar Ming the Chinese given name"), + "given Michael; family Wong; Kar Ming the Chinese given name", + fields={'given': 'Michael', 'surname': 'Wong', 'given2': 'Kar Ming'}), Convention("Singapore", "English given name after the Chinese name", "Lim Mei Ling Jessica", "FAMILY_FIRST", "family Lim; Mei Ling the Chinese given name; Jessica the English " - "given name"), + "given name", + fields={'surname': 'Lim', 'given': 'Mei Ling', 'given2': 'Jessica'}), Convention("Cantonese", "given name that is also a title modifier", "Wing Lau", None, - "given Wing, not a title; family Lau"), + "given Wing, not a title; family Lau", + fields={'given': 'Wing', 'surname': 'Lau'}), Convention("Hungarian", "married form on the husband's family name", "Szabóné Kiss Éva", "FAMILY_FIRST", - "Szabóné ('wife of Szabó') a married form; family Kiss; given Éva"), + "Szabóné ('wife of Szabó') a married form; family Kiss; given Éva", + unstated="no field states a married form on the husband's family name"), Convention("Afrikaans/Dutch", "capitalized particle, no given name", "Van der Walt", None, - "family Van der Walt; no given name"), + "family Van der Walt; no given name", + fields={'surname-prefix': 'Van der', 'surname-core': 'Walt'}), Convention("Arabic", "kunya ('father of')", "Abu Karim", None, "a kunya, 'father of Karim', used as a name of address; not a given " - "name plus a family name"), + "name plus a family name", + unstated="no field states a kunya"), Convention("Iranian", "-zadeh written as a separate word", "Reza Hossein Zadeh", None, "given Reza; family Hossein Zadeh, the separated spelling of " - "Hosseinzadeh"), + "Hosseinzadeh", + fields={'given': 'Reza', 'surname': 'Hossein Zadeh'}), Convention("French", "index form, elided particle after the given name", "Orval, Pierre d'", None, - "family d'Orval; given Pierre; d' belongs to the family"), + "family d'Orval; given Pierre; d' belongs to the family", + fields={'surname-prefix': "d'", 'surname-core': 'Orval', 'given': 'Pierre'}), ) diff --git a/tools/differential/conventions_agreement.json b/tools/differential/conventions_agreement.json new file mode 100644 index 00000000..3a668106 --- /dev/null +++ b/tools/differential/conventions_agreement.json @@ -0,0 +1,309 @@ +{ + "conventions/Ali Kara Oğlu/default": { + "family": { + "expected": "Kara Oğlu", + "got": "Oğlu" + }, + "given_names": { + "expected": "Ali", + "got": "Ali Kara" + }, + "text": "Ali Kara Oğlu" + }, + "conventions/Carmen Díaz Vda. de Mora/default": { + "family": { + "expected": "Díaz Vda. de Mora", + "got": "de Mora" + }, + "given_names": { + "expected": "Carmen", + "got": "Carmen Díaz Vda." + }, + "text": "Carmen Díaz Vda. de Mora" + }, + "conventions/Daw Thida/default": { + "family": { + "expected": "", + "got": "Thida" + }, + "given_names": { + "expected": "Thida", + "got": "Daw" + }, + "text": "Daw Thida", + "title": { + "expected": "Daw", + "got": "" + } + }, + "conventions/H. Budi Santoso/default": { + "family": { + "expected": "", + "got": "Santoso" + }, + "given_names": { + "expected": "Budi Santoso", + "got": "H. Budi" + }, + "text": "H. Budi Santoso", + "title": { + "expected": "H.", + "got": "" + } + }, + "conventions/Hassan B. Yusof/default": { + "family": { + "expected": "", + "got": "Yusof" + }, + "given_names": { + "expected": "Hassan B. Yusof", + "got": "Hassan B." + }, + "text": "Hassan B. Yusof" + }, + "conventions/Hassan bin Yusof/default": { + "family": { + "expected": "", + "got": "bin Yusof" + }, + "given_names": { + "expected": "Hassan bin Yusof", + "got": "Hassan" + }, + "text": "Hassan bin Yusof" + }, + "conventions/Hodan Farah Warsame/default": { + "family": { + "expected": "", + "got": "Warsame" + }, + "given_names": { + "expected": "Hodan Farah Warsame", + "got": "Hodan Farah" + }, + "text": "Hodan Farah Warsame" + }, + "conventions/K. Sivakumar/default": { + "family": { + "expected": "", + "got": "Sivakumar" + }, + "given_names": { + "expected": "K. Sivakumar", + "got": "K." + }, + "text": "K. Sivakumar" + }, + "conventions/Kavitha a/p Raman/default": { + "family": { + "expected": "", + "got": "Raman" + }, + "given_names": { + "expected": "Kavitha a/p Raman", + "got": "Kavitha a/p" + }, + "text": "Kavitha a/p Raman" + }, + "conventions/Khun Malee/default": { + "family": { + "expected": "", + "got": "Malee" + }, + "given_names": { + "expected": "Malee", + "got": "Khun" + }, + "text": "Khun Malee", + "title": { + "expected": "Khun", + "got": "" + } + }, + "conventions/Ma. Teresa Cruz Bautista/default": { + "family": { + "expected": "Cruz Bautista", + "got": "Bautista" + }, + "given_names": { + "expected": "Ma. Teresa", + "got": "Teresa Cruz" + }, + "text": "Ma. Teresa Cruz Bautista", + "title": { + "expected": "", + "got": "Ma." + } + }, + "conventions/Michael Wong Kar Ming/default": { + "family": { + "expected": "Wong", + "got": "Ming" + }, + "given_names": { + "expected": "Michael Kar Ming", + "got": "Michael Wong Kar" + }, + "text": "Michael Wong Kar Ming" + }, + "conventions/Murat Bey/default": { + "family": { + "expected": "", + "got": "Bey" + }, + "suffix": { + "expected": "Bey", + "got": "" + }, + "text": "Murat Bey" + }, + "conventions/Orval, Pierre d'/default": { + "family": { + "expected": "Orval d'", + "got": "Orval" + }, + "family_particles": { + "expected": "d'", + "got": "" + }, + "given_names": { + "expected": "Pierre", + "got": "Pierre d'" + }, + "text": "Orval, Pierre d'" + }, + "conventions/Park Ji-hoon ssi/FAMILY_FIRST": { + "given_names": { + "expected": "Ji-hoon", + "got": "Ji-hoon ssi" + }, + "suffix": { + "expected": "ssi", + "got": "" + }, + "text": "Park Ji-hoon ssi" + }, + "conventions/Pedro López Ruiz H./default": { + "family": { + "expected": "López Ruiz", + "got": "H." + }, + "given_names": { + "expected": "Pedro", + "got": "Pedro López Ruiz" + }, + "suffix": { + "expected": "H.", + "got": "" + }, + "text": "Pedro López Ruiz H." + }, + "conventions/Reza Hossein Zadeh/default": { + "family": { + "expected": "Hossein Zadeh", + "got": "Zadeh" + }, + "given_names": { + "expected": "Reza", + "got": "Reza Hossein" + }, + "text": "Reza Hossein Zadeh" + }, + "conventions/Suresh a/l Muthu/default": { + "family": { + "expected": "", + "got": "Muthu" + }, + "given_names": { + "expected": "Suresh a/l Muthu", + "got": "Suresh a/l" + }, + "text": "Suresh a/l Muthu" + }, + "conventions/Suzuki san/FAMILY_FIRST": { + "given_names": { + "expected": "", + "got": "san" + }, + "suffix": { + "expected": "san", + "got": "" + }, + "text": "Suzuki san" + }, + "conventions/Suzuki-san/FAMILY_FIRST": { + "family": { + "expected": "Suzuki", + "got": "Suzuki-san" + }, + "suffix": { + "expected": "-san", + "got": "" + }, + "text": "Suzuki-san" + }, + "conventions/Szabó Péter úr/FAMILY_FIRST": { + "given_names": { + "expected": "Péter", + "got": "Péter úr" + }, + "suffix": { + "expected": "úr", + "got": "" + }, + "text": "Szabó Péter úr" + }, + "conventions/Tran Thi Mai/FAMILY_FIRST": { + "given_names": { + "expected": "Thi Mai", + "got": "Thi" + }, + "suffix": { + "expected": "", + "got": "Mai" + }, + "text": "Tran Thi Mai" + }, + "conventions/U Kyaw Win/default": { + "family": { + "expected": "", + "got": "Win" + }, + "given_names": { + "expected": "Kyaw Win", + "got": "U Kyaw" + }, + "text": "U Kyaw Win", + "title": { + "expected": "U", + "got": "" + } + }, + "conventions/Van der Walt/default": { + "family": { + "expected": "Van der Walt", + "got": "der Walt" + }, + "family_particles": { + "expected": "Van der", + "got": "der" + }, + "given_names": { + "expected": "", + "got": "Van" + }, + "text": "Van der Walt" + }, + "conventions/Wing Lau/default": { + "given_names": { + "expected": "Wing", + "got": "" + }, + "text": "Wing Lau", + "title": { + "expected": "", + "got": "Wing" + } + } +} diff --git a/tools/differential/conventions_fields.jsonl b/tools/differential/conventions_fields.jsonl new file mode 100644 index 00000000..3ab3ea75 --- /dev/null +++ b/tools/differential/conventions_fields.jsonl @@ -0,0 +1,30 @@ +{"fields": {"given": "Hodan", "patronymic": "Farah Warsame"}, "file": "conventions", "label": "Hodan Farah Warsame", "order": "default", "shape": null, "text": "Hodan Farah Warsame"} +{"fields": {"given": "Harpreet", "given2": "Kaur", "surname": "Dhillon"}, "file": "conventions", "label": "Harpreet Kaur Dhillon", "order": "default", "shape": null, "text": "Harpreet Kaur Dhillon"} +{"fields": {"given": "Sivakumar", "patronymic": "K."}, "file": "conventions", "label": "K. Sivakumar", "order": "default", "shape": null, "text": "K. Sivakumar"} +{"fields": {"generation": "H.", "given": "Pedro", "surname": "López", "surname2": "Ruiz"}, "file": "conventions", "label": "Pedro López Ruiz H.", "order": "default", "shape": null, "text": "Pedro López Ruiz H."} +{"fields": {"given": "Carmen", "surname": "Díaz Vda. de Mora"}, "file": "conventions", "label": "Carmen Díaz Vda. de Mora", "order": "default", "shape": null, "text": "Carmen Díaz Vda. de Mora"} +{"fields": {"given": "Ali", "surname": "Kara Oğlu"}, "file": "conventions", "label": "Ali Kara Oğlu", "order": "default", "shape": null, "text": "Ali Kara Oğlu"} +{"fields": {"given": "Murat", "honorific": "Bey"}, "file": "conventions", "label": "Murat Bey", "order": "default", "shape": null, "text": "Murat Bey"} +{"fields": {"given": "Péter", "honorific": "úr", "surname": "Szabó"}, "file": "conventions", "label": "Szabó Péter úr", "order": "FAMILY_FIRST", "shape": 4, "text": "Szabó Péter úr"} +{"fields": {"given": "Péterné", "surname": "Szabó"}, "file": "conventions", "label": "Szabó Péterné", "order": "FAMILY_FIRST", "shape": 4, "text": "Szabó Péterné"} +{"fields": {"given": "Min Ho", "surname": "Kim"}, "file": "conventions", "label": "Kim Min Ho", "order": "FAMILY_FIRST", "shape": 4, "text": "Kim Min Ho"} +{"fields": {"given": "Mai", "given2": "Thi", "surname": "Tran"}, "file": "conventions", "label": "Tran Thi Mai", "order": "FAMILY_FIRST", "shape": 4, "text": "Tran Thi Mai"} +{"fields": {"given": "Ivan", "patronymic": "Petrov", "surname": "Dimitrov"}, "file": "conventions", "label": "Ivan Petrov Dimitrov", "order": "default", "shape": null, "text": "Ivan Petrov Dimitrov"} +{"fields": {"given": "Ma. Teresa", "surname": "Bautista", "surname2": "Cruz"}, "file": "conventions", "label": "Ma. Teresa Cruz Bautista", "order": "default", "shape": null, "text": "Ma. Teresa Cruz Bautista"} +{"fields": {"given": "Ji-hoon", "honorific": "ssi", "surname": "Park"}, "file": "conventions", "label": "Park Ji-hoon ssi", "order": "FAMILY_FIRST", "shape": 4, "text": "Park Ji-hoon ssi"} +{"fields": {"honorific": "san", "surname": "Suzuki"}, "file": "conventions", "label": "Suzuki san", "order": "FAMILY_FIRST", "shape": 4, "text": "Suzuki san"} +{"fields": {"honorific": "-san", "surname": "Suzuki"}, "file": "conventions", "label": "Suzuki-san", "order": "FAMILY_FIRST", "shape": 4, "text": "Suzuki-san"} +{"fields": {"given": "Kyaw Win", "title": "U"}, "file": "conventions", "label": "U Kyaw Win", "order": "default", "shape": null, "text": "U Kyaw Win"} +{"fields": {"given": "Thida", "title": "Daw"}, "file": "conventions", "label": "Daw Thida", "order": "default", "shape": null, "text": "Daw Thida"} +{"fields": {"given": "Malee", "title": "Khun"}, "file": "conventions", "label": "Khun Malee", "order": "default", "shape": null, "text": "Khun Malee"} +{"fields": {"given": "Hassan", "patronymic": "bin Yusof"}, "file": "conventions", "label": "Hassan bin Yusof", "order": "default", "shape": null, "text": "Hassan bin Yusof"} +{"fields": {"given": "Hassan", "patronymic": "B. Yusof"}, "file": "conventions", "label": "Hassan B. Yusof", "order": "default", "shape": null, "text": "Hassan B. Yusof"} +{"fields": {"given": "Suresh", "patronymic": "a/l Muthu"}, "file": "conventions", "label": "Suresh a/l Muthu", "order": "default", "shape": null, "text": "Suresh a/l Muthu"} +{"fields": {"given": "Kavitha", "patronymic": "a/p Raman"}, "file": "conventions", "label": "Kavitha a/p Raman", "order": "default", "shape": null, "text": "Kavitha a/p Raman"} +{"fields": {"given": "Budi Santoso", "title": "H."}, "file": "conventions", "label": "H. Budi Santoso", "order": "default", "shape": null, "text": "H. Budi Santoso"} +{"fields": {"given": "Michael", "given2": "Kar Ming", "surname": "Wong"}, "file": "conventions", "label": "Michael Wong Kar Ming", "order": "default", "shape": null, "text": "Michael Wong Kar Ming"} +{"fields": {"given": "Mei Ling", "given2": "Jessica", "surname": "Lim"}, "file": "conventions", "label": "Lim Mei Ling Jessica", "order": "FAMILY_FIRST", "shape": 4, "text": "Lim Mei Ling Jessica"} +{"fields": {"given": "Wing", "surname": "Lau"}, "file": "conventions", "label": "Wing Lau", "order": "default", "shape": null, "text": "Wing Lau"} +{"fields": {"surname-core": "Walt", "surname-prefix": "Van der"}, "file": "conventions", "label": "Van der Walt", "order": "default", "shape": null, "text": "Van der Walt"} +{"fields": {"given": "Reza", "surname": "Hossein Zadeh"}, "file": "conventions", "label": "Reza Hossein Zadeh", "order": "default", "shape": null, "text": "Reza Hossein Zadeh"} +{"fields": {"given": "Pierre", "surname-core": "Orval", "surname-prefix": "d'"}, "file": "conventions", "label": "Orval, Pierre d'", "order": "default", "shape": null, "text": "Orval, Pierre d'"}