From 98daf55348a968ac57b4893d9a4b211bb11082e3 Mon Sep 17 00:00:00 2001 From: Leon-L20 Date: Fri, 18 Sep 2026 08:40:04 +0000 Subject: [PATCH 1/9] =?UTF-8?q?feat(pms):=20prompt=20profile=20=E6=9C=BA?= =?UTF-8?q?=E5=88=B6=20+=20=E8=BE=93=E5=87=BA=E9=97=A8/=E5=B9=B6=E5=8F=91?= =?UTF-8?q?=E5=B0=81=E9=A1=B6=E7=BB=84=E4=BB=B6=E5=8C=96=20+=20=E5=A4=8D?= =?UTF-8?q?=E4=BD=8D=E4=B8=A2=E5=A4=B1=E7=9A=84=E5=81=A5=E5=A3=AE=E6=80=A7?= =?UTF-8?q?=E8=A1=A5=E4=B8=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增版本化 prompt profile 加载器（结构 fail-closed 校验），10 个 generate method 全部支持 config 级模板覆盖 - pms_policy.v2 内置 profile：迁移 PMS 侧 v2 出题策略模板为正本 - 输出门注册表（pms_project_anchor）与 LLM 并发封顶成为 GenerateService 一等参数 - 复位历史上随 checkout 重建丢失的三个补丁：空响应重试、空抽取 prefill、logprob 调用关思考（env 开关语义不变） - 台账见 docs/PMS_PATCHES.md --- docs/PMS_PATCHES.md | 64 ++++++++++ graphgen/bases/base_generator.py | 40 +++++- .../extractor/schema_guided_extractor.py | 14 +++ .../models/generator/aggregated_generator.py | 14 +-- graphgen/models/generator/atomic_generator.py | 9 +- graphgen/models/generator/cot_generator.py | 13 +- .../generator/fill_in_blank_generator.py | 4 +- .../masked_fill_in_blank_generator.py | 9 +- .../generator/multi_answer_generator.py | 4 +- .../generator/multi_choice_generator.py | 4 +- .../models/generator/multi_hop_generator.py | 9 +- .../models/generator/true_false_generator.py | 4 +- graphgen/models/generator/vqa_generator.py | 8 +- graphgen/models/llm/api/openai_client.py | 70 +++++++++-- graphgen/operators/generate/gates.py | 53 ++++++++ .../operators/generate/generate_service.py | 32 +++++ .../operators/generate/llm_concurrency.py | 33 +++++ .../profiles/pms_policy.v2/atomic.json | 4 + .../profiles/pms_policy.v2/multi_hop.json | 4 + graphgen/templates/prompt_profiles.py | 119 ++++++++++++++++++ 20 files changed, 460 insertions(+), 51 deletions(-) create mode 100644 docs/PMS_PATCHES.md create mode 100644 graphgen/operators/generate/gates.py create mode 100644 graphgen/operators/generate/llm_concurrency.py create mode 100644 graphgen/templates/profiles/pms_policy.v2/atomic.json create mode 100644 graphgen/templates/profiles/pms_policy.v2/multi_hop.json create mode 100644 graphgen/templates/prompt_profiles.py diff --git a/docs/PMS_PATCHES.md b/docs/PMS_PATCHES.md new file mode 100644 index 00000000..0d1ee4a9 --- /dev/null +++ b/docs/PMS_PATCHES.md @@ -0,0 +1,64 @@ +# PMS Fork 补丁台账 + +本仓库是 `open-sciencelab/GraphGen`（upstream，Apache-2.0）的自维护 fork，分支 +`pms/main`。上游修复可通过 rebase 同步，冲突以本 fork 语义为准。PMS 主仓库只钉 +住本 fork 的 commit SHA（receipt `_graphgen_source_identity` 自动记录）。 + +## #1 版本化 Prompt Profile（2026-09-18） + +- **动机**：官方所有 prompt 是 `graphgen/templates/` 下的硬编码常量，无任何 + config 覆盖机制；此前 PMS 侧只能 monkeypatch `generator.build_prompt`。 +- **实现**： + - `graphgen/templates/prompt_profiles.py`：profile 加载器。profile = 内置 id + （`graphgen/templates/profiles//`）或文件路径；每 method 一个 + `.json`；结构与官方模板逐键校验（fail-closed）。 + - `graphgen/bases/base_generator.py`：`apply_prompt_profile()` + + `template()`；generators 声明 `TEMPLATE_KEY` 并经 `self.template(...)` + 渲染；未应用 profile 时与官方常量逐字节一致。 + - 10 个 generate method（atomic/aggregated/multi_hop/cot/multi_choice/ + multi_answer/fill_in_blank/masked_fill_in_blank/true_false/vqa）全部接入。 + - `graphgen/operators/generate/generate_service.py`：generate 节点新增 + `prompt_profile` / `output_gate` / `llm_concurrency` 三个可选参数。 +- **内置 profile**：`pms_policy.v2`（atomic + multi_hop 的 PMS project-query + 出题策略模板，正本迁移自 PMS 受控侧 v2 模板）。 + +## #2 确定性输出门（2026-09-18） + +- `graphgen/operators/generate/gates.py`：注册表 + `pms_project_anchor` 门 + （题面必须含完整项目编号 P-/I-+≥6 位且答案非空，否则丢弃该 QA，不改写）。 +- generate 节点经 `output_gate` 参数启用。 + +## #3 生成 LLM 并发封顶（2026-09-18） + +- `graphgen/operators/generate/llm_concurrency.py`：事件循环级信号量， + 把同 loop 在途 generate 压到上限（35B-A3B 高并发实测会过载挂死）。 +- generate 节点经 `llm_concurrency` 参数启用（PMS 默认 4）。 + +## #4 空响应有界重试（复位自丢失补丁，2026-09-18） + +- `graphgen/models/llm/api/openai_client.py::generate_answer`：思考模型偶发 + 仅返回 `` 内容、过滤后为空串；`GRAPHGEN_EMPTY_RETRY=N`（默认 0， + 行为同官方）时对空响应做最多 N 次重试。历史版本此补丁仅存于 env 注入、 + 源码实现已丢失，本条为正确复位。 + +## #5 空抽取 assistant prefill 重试（复位自丢失补丁，2026-09-18） + +- `openai_client.generate_answer` 新增 `assistant_prefill` 参数； + `schema_guided_extractor.extract` 在响应为空且 + `GRAPHGEN_EMPTY_EXTRACTION_PREFILL=1` 时，以确定性 prefill `("entity"<|>` + 重试一次。非空响应永不重试/改写。 + +## #6 logprob 调用关思考（复位自丢失补丁，2026-09-18） + +- `generate_topk_per_token`：思考模型前若干 token 消耗在 `` 内，导致 + judge/loss 的 next-token yes/no logprob 退化。`GRAPHGEN_DISABLE_THINKING=1` + 时对 logprob 调用注入 `extra_body.chat_template_kwargs.enable_thinking=false` + （Qwen 兼容后端）。普通生成调用不受影响。 + +## 验收 + +- 默认路径（无 profile/gate/并发参数）与官方行为逐字节一致； +- PMS 主仓库 `tests/test_pms_graphgen_policy_generation.py`、 + `tests/test_run_graphgen_defaults.py` 为契约回归（rft 环境按路径加载 + 纯 stdlib 模块校验）； +- 真实生成由 GraphGen venv smoke 验收。 diff --git a/graphgen/bases/base_generator.py b/graphgen/bases/base_generator.py index 42e55180..04867b88 100644 --- a/graphgen/bases/base_generator.py +++ b/graphgen/bases/base_generator.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import Any +from typing import Any, Optional from graphgen.bases.base_llm_wrapper import BaseLLMWrapper @@ -7,10 +7,48 @@ class BaseGenerator(ABC): """ Generate QAs based on given prompts. + + PMS fork: prompt templates can be overridden per instance via a versioned + prompt profile (see graphgen.templates.prompt_profiles). Generators declare + their official template under ``TEMPLATE_KEY`` and render through + ``self.template(...)``; without a profile the official constant is used + byte-for-byte. """ + #: key into graphgen.templates.prompt_profiles.OFFICIAL_TEMPLATES + TEMPLATE_KEY: Optional[str] = None + def __init__(self, llm_client: BaseLLMWrapper): self.llm_client = llm_client + self._profile_templates: Optional[dict] = None + + def apply_prompt_profile(self, profile: str) -> None: + """Attach a versioned prompt profile to this generator instance.""" + from graphgen.templates.prompt_profiles import load_profile_template + + if self.TEMPLATE_KEY is None: + raise ValueError( + f"prompt_profile_unsupported_generator:{type(self).__name__}" + ) + self._profile_templates = load_profile_template(profile, self.TEMPLATE_KEY) + + def template(self, language: str, key: Optional[str] = None) -> str: + """Resolve the effective template for this generator. + + :param language: detected language key ("en" / "zh"). + :param key: nested sub-template key for multi-phase generators. + """ + from graphgen.templates.prompt_profiles import OFFICIAL_TEMPLATES + + if self.TEMPLATE_KEY is None: + raise ValueError( + f"prompt_template_key_missing:{type(self).__name__}" + ) + if self._profile_templates is not None: + node = self._profile_templates[language] + else: + node = OFFICIAL_TEMPLATES[self.TEMPLATE_KEY][language] + return node[key] if key is not None else node @staticmethod @abstractmethod diff --git a/graphgen/models/extractor/schema_guided_extractor.py b/graphgen/models/extractor/schema_guided_extractor.py index dee6e3e3..f594350f 100644 --- a/graphgen/models/extractor/schema_guided_extractor.py +++ b/graphgen/models/extractor/schema_guided_extractor.py @@ -59,10 +59,24 @@ def build_prompt(self, text: str) -> str: return prompt async def extract(self, chunk: Chunk) -> dict: + import os + text = chunk.content prompt = self.build_prompt(text) response = await self.llm_client.generate_answer(prompt) + + # PMS fork (docs/PMS_PATCHES.md #5): strict tuple extraction prompts + # occasionally hit EOS as the first token. When enabled and the first + # response is empty, retry once with a deterministic assistant prefill + # that seeds the opening of the expected output; non-empty responses + # are never retried or modified. + if not response.strip() and os.environ.get("GRAPHGEN_EMPTY_EXTRACTION_PREFILL", "0") == "1": + prefill = '("entity"<|>' + logger.warning("Empty extraction response; retrying with assistant prefill") + response = await self.llm_client.generate_answer( + prompt, assistant_prefill=prefill + ) try: extracted_info = json.loads(response) # Ensure all required keys are present diff --git a/graphgen/models/generator/aggregated_generator.py b/graphgen/models/generator/aggregated_generator.py index 3f223325..c8ef06c7 100644 --- a/graphgen/models/generator/aggregated_generator.py +++ b/graphgen/models/generator/aggregated_generator.py @@ -2,7 +2,6 @@ from typing import Any, Optional from graphgen.bases import BaseGenerator -from graphgen.templates import AGGREGATED_GENERATION_PROMPT from graphgen.utils import detect_main_language, logger @@ -14,9 +13,11 @@ class AggregatedGenerator(BaseGenerator): 2. question generation: Generate relevant questions based on the rephrased text. """ - @staticmethod + TEMPLATE_KEY = "aggregated" + def build_prompt( - batch: tuple[list[tuple[str, dict]], list[tuple[Any, Any, dict]]] + self, + batch: tuple[list[tuple[str, dict]], list[tuple[Any, Any, dict]]], ) -> str: """ Build prompts for REPHRASE. @@ -51,7 +52,7 @@ def build_prompt( # for index, text in enumerate(original_text) # ] # ) - prompt = AGGREGATED_GENERATION_PROMPT[language]["ANSWER_REPHRASING"].format( + prompt = self.template(language, "ANSWER_REPHRASING").format( entities=entities_str, relationships=relations_str ) return prompt @@ -73,15 +74,14 @@ def parse_rephrased_text(response: str) -> Optional[str]: return None return rephrased_text.strip('"').strip("'") - @staticmethod - def _build_prompt_for_question_generation(answer: str) -> str: + def _build_prompt_for_question_generation(self, answer: str) -> str: """ Build prompts for QUESTION GENERATION. :param answer: :return: """ language = detect_main_language(answer) - prompt = AGGREGATED_GENERATION_PROMPT[language]["QUESTION_GENERATION"].format( + prompt = self.template(language, "QUESTION_GENERATION").format( answer=answer ) return prompt diff --git a/graphgen/models/generator/atomic_generator.py b/graphgen/models/generator/atomic_generator.py index d045b0da..a77b731c 100644 --- a/graphgen/models/generator/atomic_generator.py +++ b/graphgen/models/generator/atomic_generator.py @@ -2,14 +2,15 @@ from typing import Any from graphgen.bases import BaseGenerator -from graphgen.templates import ATOMIC_GENERATION_PROMPT from graphgen.utils import detect_main_language, logger class AtomicGenerator(BaseGenerator): - @staticmethod + TEMPLATE_KEY = "atomic" + def build_prompt( - batch: tuple[list[tuple[str, dict]], list[tuple[Any, Any, dict]]] + self, + batch: tuple[list[tuple[str, dict]], list[tuple[Any, Any, dict]]], ) -> str: nodes, edges = batch context = "" @@ -19,7 +20,7 @@ def build_prompt( context += f"- {edge[0]} - {edge[1]}: {edge[2]['description']}\n" language = detect_main_language(context) - prompt = ATOMIC_GENERATION_PROMPT[language].format(context=context) + prompt = self.template(language).format(context=context) return prompt @staticmethod diff --git a/graphgen/models/generator/cot_generator.py b/graphgen/models/generator/cot_generator.py index 88d04324..b4edd38c 100644 --- a/graphgen/models/generator/cot_generator.py +++ b/graphgen/models/generator/cot_generator.py @@ -2,14 +2,15 @@ from typing import Any from graphgen.bases import BaseGenerator -from graphgen.templates import COT_GENERATION_PROMPT from graphgen.utils import detect_main_language, logger class CoTGenerator(BaseGenerator): - @staticmethod + TEMPLATE_KEY = "cot" + def build_prompt( - batch: tuple[list[tuple[str, dict]], list[tuple[Any, Any, dict]]] + self, + batch: tuple[list[tuple[str, dict]], list[tuple[Any, Any, dict]]], ) -> str: """ Build prompts for COT Template Design. @@ -30,13 +31,13 @@ def build_prompt( ] ) language = detect_main_language(entities_str + relationships_str) - prompt = COT_GENERATION_PROMPT[language]["COT_TEMPLATE_DESIGN"].format( + prompt = self.template(language, "COT_TEMPLATE_DESIGN").format( entities=entities_str, relationships=relationships_str ) return prompt - @staticmethod def build_prompt_for_cot_generation( + self, batch: tuple[list[tuple[str, dict]], list[tuple[Any, Any, dict]]], question: str, reasoning_path: str, @@ -58,7 +59,7 @@ def build_prompt_for_cot_generation( ] ) language = detect_main_language(entities_str + relationships_str) - prompt = COT_GENERATION_PROMPT[language]["COT_GENERATION"].format( + prompt = self.template(language, "COT_GENERATION").format( entities=entities_str, relationships=relationships_str, question=question, diff --git a/graphgen/models/generator/fill_in_blank_generator.py b/graphgen/models/generator/fill_in_blank_generator.py index a26daf3e..2d63d8d1 100644 --- a/graphgen/models/generator/fill_in_blank_generator.py +++ b/graphgen/models/generator/fill_in_blank_generator.py @@ -2,11 +2,11 @@ from typing import Any from graphgen.bases import BaseGenerator -from graphgen.templates import FILL_IN_BLANK_GENERATION_PROMPT from graphgen.utils import detect_main_language, logger class FillInBlankGenerator(BaseGenerator): + TEMPLATE_KEY = "fill_in_blank" def __init__(self, llm_client, num_of_questions) -> None: super().__init__(llm_client) self.num_of_questions = num_of_questions @@ -92,7 +92,7 @@ def build_prompt( ) context = entities_str + "\n" + relationships_str language = detect_main_language(entities_str + relationships_str) - prompt = FILL_IN_BLANK_GENERATION_PROMPT[language].format( + prompt = self.template(language).format( context=context, num_of_questions=self.num_of_questions, ) diff --git a/graphgen/models/generator/masked_fill_in_blank_generator.py b/graphgen/models/generator/masked_fill_in_blank_generator.py index 355cadcf..ab941688 100644 --- a/graphgen/models/generator/masked_fill_in_blank_generator.py +++ b/graphgen/models/generator/masked_fill_in_blank_generator.py @@ -3,7 +3,6 @@ from typing import Any, Optional from graphgen.bases import BaseGenerator -from graphgen.templates import AGGREGATED_GENERATION_PROMPT from graphgen.utils import detect_main_language, logger random.seed(42) @@ -16,9 +15,11 @@ class MaskedFillInBlankGenerator(BaseGenerator): 2. mask: Randomly select a node from the input nodes, and then mask the name of the node in the rephrased text. """ - @staticmethod + TEMPLATE_KEY = "masked_fill_in_blank" + def build_prompt( - batch: tuple[list[tuple[str, dict]], list[tuple[Any, Any, dict]]] + self, + batch: tuple[list[tuple[str, dict]], list[tuple[Any, Any, dict]]], ) -> str: """ Build prompts for REPHRASE. @@ -53,7 +54,7 @@ def build_prompt( # for index, text in enumerate(original_text) # ] # ) - prompt = AGGREGATED_GENERATION_PROMPT[language]["ANSWER_REPHRASING"].format( + prompt = self.template(language, "ANSWER_REPHRASING").format( entities=entities_str, relationships=relations_str ) return prompt diff --git a/graphgen/models/generator/multi_answer_generator.py b/graphgen/models/generator/multi_answer_generator.py index a341a4fd..c9af5701 100644 --- a/graphgen/models/generator/multi_answer_generator.py +++ b/graphgen/models/generator/multi_answer_generator.py @@ -2,11 +2,11 @@ from typing import Any from graphgen.bases import BaseGenerator -from graphgen.templates import MAQ_GENERATION_PROMPT from graphgen.utils import detect_main_language, logger class MultiAnswerGenerator(BaseGenerator): + TEMPLATE_KEY = "multi_answer" def __init__(self, llm_client, num_of_questions) -> None: super().__init__(llm_client) self.num_of_questions = num_of_questions @@ -113,7 +113,7 @@ def build_prompt( ) context = entities_str + "\n" + relationships_str language = detect_main_language(entities_str + relationships_str) - prompt = MAQ_GENERATION_PROMPT[language].format( + prompt = self.template(language).format( context=context, num_of_questions=self.num_of_questions, ) diff --git a/graphgen/models/generator/multi_choice_generator.py b/graphgen/models/generator/multi_choice_generator.py index 0c48b76d..3a16a550 100644 --- a/graphgen/models/generator/multi_choice_generator.py +++ b/graphgen/models/generator/multi_choice_generator.py @@ -2,11 +2,11 @@ from typing import Any from graphgen.bases import BaseGenerator -from graphgen.templates import MCQ_GENERATION_PROMPT from graphgen.utils import detect_main_language, logger class MultiChoiceGenerator(BaseGenerator): + TEMPLATE_KEY = "multi_choice" def __init__(self, llm_client, num_of_questions) -> None: super().__init__(llm_client) self.num_of_questions = num_of_questions @@ -111,7 +111,7 @@ def build_prompt( ) context = entities_str + "\n" + relationships_str language = detect_main_language(entities_str + relationships_str) - prompt = MCQ_GENERATION_PROMPT[language].format( + prompt = self.template(language).format( context=context, num_of_questions=self.num_of_questions, ) diff --git a/graphgen/models/generator/multi_hop_generator.py b/graphgen/models/generator/multi_hop_generator.py index a19082b9..232f0e56 100644 --- a/graphgen/models/generator/multi_hop_generator.py +++ b/graphgen/models/generator/multi_hop_generator.py @@ -2,14 +2,15 @@ from typing import Any from graphgen.bases import BaseGenerator -from graphgen.templates import MULTI_HOP_GENERATION_PROMPT from graphgen.utils import detect_main_language, logger class MultiHopGenerator(BaseGenerator): - @staticmethod + TEMPLATE_KEY = "multi_hop" + def build_prompt( - batch: tuple[list[tuple[str, dict]], list[tuple[Any, Any, dict]]] + self, + batch: tuple[list[tuple[str, dict]], list[tuple[Any, Any, dict]]], ) -> str: nodes, edges = batch entities_str = "\n".join( @@ -26,7 +27,7 @@ def build_prompt( ] ) language = detect_main_language(entities_str + relationships_str) - prompt = MULTI_HOP_GENERATION_PROMPT[language].format( + prompt = self.template(language).format( entities=entities_str, relationships=relationships_str ) return prompt diff --git a/graphgen/models/generator/true_false_generator.py b/graphgen/models/generator/true_false_generator.py index 1a1fa0d3..c51fe371 100644 --- a/graphgen/models/generator/true_false_generator.py +++ b/graphgen/models/generator/true_false_generator.py @@ -2,11 +2,11 @@ from typing import Any from graphgen.bases import BaseGenerator -from graphgen.templates import TF_GENERATION_PROMPT from graphgen.utils import detect_main_language, logger class TrueFalseGenerator(BaseGenerator): + TEMPLATE_KEY = "true_false" def __init__(self, llm_client, num_of_questions) -> None: super().__init__(llm_client) self.num_of_questions = num_of_questions @@ -84,7 +84,7 @@ def build_prompt( ) context = entities_str + "\n" + relationships_str language = detect_main_language(entities_str + relationships_str) - prompt = TF_GENERATION_PROMPT[language].format( + prompt = self.template(language).format( context=context, num_of_questions=self.num_of_questions, ) diff --git a/graphgen/models/generator/vqa_generator.py b/graphgen/models/generator/vqa_generator.py index 723bd2a6..a6375a10 100644 --- a/graphgen/models/generator/vqa_generator.py +++ b/graphgen/models/generator/vqa_generator.py @@ -3,14 +3,14 @@ from typing import Any from graphgen.bases import BaseGenerator -from graphgen.templates import VQA_GENERATION_PROMPT from graphgen.utils import detect_main_language, logger class VQAGenerator(BaseGenerator): - @staticmethod + TEMPLATE_KEY = "vqa" def build_prompt( - batch: tuple[list[tuple[str, dict]], list[tuple[Any, Any, dict]]] + self, + batch: tuple[list[tuple[str, dict]], list[tuple[Any, Any, dict]]], ) -> str: nodes, edges = batch entities_str = "\n".join( @@ -27,7 +27,7 @@ def build_prompt( ] ) language = detect_main_language(entities_str + relationships_str) - prompt = VQA_GENERATION_PROMPT[language].format( + prompt = self.template(language).format( entities=entities_str, relationships=relationships_str ) return prompt diff --git a/graphgen/models/llm/api/openai_client.py b/graphgen/models/llm/api/openai_client.py index 532b981c..916424af 100644 --- a/graphgen/models/llm/api/openai_client.py +++ b/graphgen/models/llm/api/openai_client.py @@ -13,6 +13,7 @@ from graphgen.bases.base_llm_wrapper import BaseLLMWrapper from graphgen.bases.datatypes import Token from graphgen.models.llm.limitter import RPM, TPM +from graphgen.utils import logger def get_top_response_tokens(response: openai.ChatCompletion) -> List[Token]: @@ -86,7 +87,9 @@ def __post_init__(self): else: raise ValueError(f"Unsupported backend {self.backend}. Use 'openai_api' or 'azure_openai_api'.") - def _pre_generate(self, text: str, history: List[str]) -> Dict: + def _pre_generate( + self, text: str, history: List[str], assistant_prefill: Optional[str] = None + ) -> Dict: kwargs = { "temperature": self.temperature, "top_p": self.top_p, @@ -106,6 +109,11 @@ def _pre_generate(self, text: str, history: List[str]) -> Dict: assert len(history) % 2 == 0, "History should have even number of elements." messages = history + messages + # PMS fork (docs/PMS_PATCHES.md #5): deterministic assistant prefill for + # strict-prompt backends that occasionally emit EOS as the first token. + if assistant_prefill is not None: + messages.append({"role": "assistant", "content": assistant_prefill}) + kwargs["messages"] = messages return kwargs @@ -130,6 +138,18 @@ async def generate_topk_per_token( # Limit max_tokens to 1 to avoid long completions kwargs["max_tokens"] = 1 + # PMS fork (docs/PMS_PATCHES.md #6): thinking models spend the first + # tokens inside content, which makes next-token yes/no logprobs + # meaningless for judge/loss calls. GRAPHGEN_DISABLE_THINKING=1 asks the + # backend to disable thinking for these logprob-based calls only. + import os + + if os.environ.get("GRAPHGEN_DISABLE_THINKING", "0") == "1": + kwargs["extra_body"] = { + **kwargs.get("extra_body", {}), + "chat_template_kwargs": {"enable_thinking": False}, + } + completion = await self.client.chat.completions.create( # pylint: disable=E1125 model=self.model, **kwargs ) @@ -149,9 +169,10 @@ async def generate_answer( self, text: str, history: Optional[List[str]] = None, + assistant_prefill: Optional[str] = None, **extra: Any, ) -> str: - kwargs = self._pre_generate(text, history) + kwargs = self._pre_generate(text, history, assistant_prefill) prompt_tokens = 0 for message in kwargs["messages"]: @@ -162,18 +183,41 @@ async def generate_answer( await self.rpm.wait(silent=True) await self.tpm.wait(estimated_tokens, silent=True) - completion = await self.client.chat.completions.create( # pylint: disable=E1125 - model=self.model, **kwargs - ) - if hasattr(completion, "usage"): - self.token_usage.append( - { - "prompt_tokens": completion.usage.prompt_tokens, - "completion_tokens": completion.usage.completion_tokens, - "total_tokens": completion.usage.total_tokens, - } + # PMS fork (docs/PMS_PATCHES.md #4): bounded retry on empty responses. + # Thinking backends occasionally return only content that is + # stripped to nothing downstream; retry instead of propagating an empty + # string that would fail parsing. GRAPHGEN_EMPTY_RETRY=0 keeps official + # single-attempt behavior. + import os + + try: + empty_retries = max(0, int(os.environ.get("GRAPHGEN_EMPTY_RETRY", "0"))) + except ValueError: + empty_retries = 0 + + content = "" + for attempt in range(empty_retries + 1): + completion = await self.client.chat.completions.create( # pylint: disable=E1125 + model=self.model, **kwargs + ) + if hasattr(completion, "usage"): + self.token_usage.append( + { + "prompt_tokens": completion.usage.prompt_tokens, + "completion_tokens": completion.usage.completion_tokens, + "total_tokens": completion.usage.total_tokens, + } + ) + content = self.filter_think_tags(completion.choices[0].message.content) + if content.strip(): + break + logger.warning( + "Empty LLM response (attempt %d/%d, model=%s)", + attempt + 1, + empty_retries + 1, + self.model, ) - return self.filter_think_tags(completion.choices[0].message.content) + return content async def generate_inputs_prob( self, text: str, history: Optional[List[str]] = None, **extra: Any diff --git a/graphgen/operators/generate/gates.py b/graphgen/operators/generate/gates.py new file mode 100644 index 00000000..63eb254a --- /dev/null +++ b/graphgen/operators/generate/gates.py @@ -0,0 +1,53 @@ +"""Deterministic output gates for generated QA pairs (PMS fork capability). + +A gate is a pure, synchronous filter applied to the parsed QA list of one +generate call before results reach the native cache. Gates never modify or +fabricate model output; they only drop pairs that violate a declared +deterministic contract. See docs/PMS_PATCHES.md #2. +""" + +from __future__ import annotations + +import re +from typing import Any, Callable + +GATES: dict[str, Callable[[Any], list[dict]]] = {} + +_PROJECT_ID_PATTERN = re.compile(r"(?:P|I)-\d{6,}") + + +def register_gate(name: str) -> Callable: + def _decorator(func: Callable[[Any], list[dict]]) -> Callable[[Any], list[dict]]: + if name in GATES: + raise ValueError(f"output_gate_already_registered:{name}") + GATES[name] = func + return func + return _decorator + + +@register_gate("pms_project_anchor") +def pms_project_anchor(qa_pairs: Any) -> list[dict]: + """PMS project-query anchor gate. + + Only pairs whose question carries a complete project id (P-/I- plus at + least 6 digits) and a non-empty answer are executable by the project-query + skill; name-only or refusal placeholder questions are dropped. + """ + if not isinstance(qa_pairs, list): + return [] + return [ + pair + for pair in qa_pairs + if isinstance(pair, dict) + and isinstance(pair.get("question"), str) + and bool(_PROJECT_ID_PATTERN.search(pair["question"])) + and isinstance(pair.get("answer"), str) + and pair["answer"].strip() + ] + + +def apply_output_gate(name: str, qa_pairs: Any) -> list[dict]: + """Apply a registered gate by name; fail-closed on unknown gates.""" + if name not in GATES: + raise ValueError(f"output_gate_not_registered:{name}") + return GATES[name](qa_pairs) diff --git a/graphgen/operators/generate/generate_service.py b/graphgen/operators/generate/generate_service.py index 18ce1d43..52359b29 100644 --- a/graphgen/operators/generate/generate_service.py +++ b/graphgen/operators/generate/generate_service.py @@ -9,6 +9,13 @@ class GenerateService(BaseOperator): """ Generate question-answer pairs based on nodes and edges. + + PMS fork additions (all optional, default = official behavior): + - ``prompt_profile``: versioned prompt override loaded via + graphgen.templates.prompt_profiles; + - ``output_gate``: deterministic QA filter registered in + graphgen.operators.generate.gates; + - ``llm_concurrency``: cap on in-flight LLM requests per event loop. """ def __init__( @@ -17,6 +24,9 @@ def __init__( kv_backend: str = "rocksdb", method: str = "aggregated", data_format: str = "ChatML", + prompt_profile: str | None = None, + output_gate: str | None = None, + llm_concurrency: int | None = None, **generate_kwargs, ): super().__init__( @@ -85,6 +95,28 @@ def __init__( else: raise ValueError(f"Unsupported generation mode: {method}") + if prompt_profile: + self.generator.apply_prompt_profile(prompt_profile) + if llm_concurrency is not None: + from graphgen.operators.generate.llm_concurrency import ( + bound_llm_concurrency, + ) + + limited = bound_llm_concurrency(self.generator.generate, llm_concurrency) + self.generator.generate = limited + if output_gate: + from graphgen.operators.generate.gates import GATES + + if output_gate not in GATES: + raise ValueError(f"output_gate_not_registered:{output_gate}") + gate = GATES[output_gate] + base_generate = self.generator.generate + + async def _gated_generate(batch: list) -> list[dict]: + return gate(await base_generate(batch)) + + self.generator.generate = _gated_generate + def process(self, batch: list) -> Tuple[list, dict]: """ Generate question-answer pairs based on nodes and edges. diff --git a/graphgen/operators/generate/llm_concurrency.py b/graphgen/operators/generate/llm_concurrency.py new file mode 100644 index 00000000..685809e6 --- /dev/null +++ b/graphgen/operators/generate/llm_concurrency.py @@ -0,0 +1,33 @@ +"""Bound in-flight LLM requests of a generator inside one event loop. + +Official ``run_concurrent`` has no cross-batch concurrency cap; a busy LLM +endpoint (e.g. a 35B-A3B deployment) can deadlock under high fan-out. This +helper caps concurrent ``generator.generate`` calls with an asyncio.Semaphore +created lazily per running loop (Ray actors create a fresh loop per batch). +Pure stdlib so PMS-side contract tests can import this module by path without +the graphgen package installed. See docs/PMS_PATCHES.md #3. +""" + +from __future__ import annotations + +import asyncio +from typing import Any + + +def bound_llm_concurrency(generate: Any, limit: int) -> Any: + """Wrap an async ``generate(batch)`` so at most ``limit`` run in-flight.""" + if limit < 1: + raise ValueError("llm_concurrency_must_be_positive") + state: dict[str, Any] = {} + + async def _gated_generate(batch: Any) -> Any: + loop = asyncio.get_running_loop() + semaphore = state.get("semaphore") + if semaphore is None or state.get("loop") is not loop: + semaphore = asyncio.Semaphore(limit) + state["semaphore"] = semaphore + state["loop"] = loop + async with semaphore: + return await generate(batch) + + return _gated_generate diff --git a/graphgen/templates/profiles/pms_policy.v2/atomic.json b/graphgen/templates/profiles/pms_policy.v2/atomic.json new file mode 100644 index 00000000..ff6b8411 --- /dev/null +++ b/graphgen/templates/profiles/pms_policy.v2/atomic.json @@ -0,0 +1,4 @@ +{ + "en": "你是 PMS 项目管理系统的训练题生成器。给定知识图谱片段（节点与关系描述），生成一个可直接执行的“项目查询”问答题。\n\n出题铁律：\n1. 【单项目锚点】问题必须以一个明确项目为对象：题面直接包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。禁止使用“该项目/本项目/该工程”等模糊指代。\n2. 【字段锁定】只允许询问以下信息：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n3. 【禁止反查】不得以人员、部门、组织为起点提问（如“某人属于哪个部门”“某部门有哪些人”“某公司在项目中扮演什么角色”）；人员只能作为“某项目成员名单/某项目某角色由谁担任”的被问内容。\n4. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、管理效果、分工、阶段码含义、组织关系或跨项目比较。\n5. 【事实可溯】答案必须完全来自给定图谱内容；答案给出图谱中的具体值，不得编造。\n6. 【问法自然】使用中文业务口吻提问，问题应像一个真实用户向项目管理系统发起的查询。\n\n输出格式：\nquestion_text\nanswer_text\n\n给定图谱片段：\n{context}\n\n输出：\n", + "zh": "你是 PMS 项目管理系统的训练题生成器。给定知识图谱片段（节点与关系描述），生成一个可直接执行的“项目查询”问答题。\n\n出题铁律：\n1. 【单项目锚点】问题必须以一个明确项目为对象：题面直接包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。禁止使用“该项目/本项目/该工程”等模糊指代。\n2. 【字段锁定】只允许询问以下信息：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n3. 【禁止反查】不得以人员、部门、组织为起点提问（如“某人属于哪个部门”“某部门有哪些人”“某公司在项目中扮演什么角色”）；人员只能作为“某项目成员名单/某项目某角色由谁担任”的被问内容。\n4. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、管理效果、分工、阶段码含义、组织关系或跨项目比较。\n5. 【事实可溯】答案必须完全来自给定图谱内容；答案给出图谱中的具体值，不得编造。\n6. 【问法自然】使用中文业务口吻提问，问题应像一个真实用户向项目管理系统发起的查询。\n\n输出格式：\nquestion_text\nanswer_text\n\n给定图谱片段：\n{context}\n\n输出：\n" +} diff --git a/graphgen/templates/profiles/pms_policy.v2/multi_hop.json b/graphgen/templates/profiles/pms_policy.v2/multi_hop.json new file mode 100644 index 00000000..bd6798a0 --- /dev/null +++ b/graphgen/templates/profiles/pms_policy.v2/multi_hop.json @@ -0,0 +1,4 @@ +{ + "en": "你是 PMS 项目管理系统的训练题生成器。给定一个知识子图（实体与关系），生成一个需要两步及以上查询、但仍可由 project-query 技能逐步执行的中文问答对。\n\n出题铁律：\n1. 【项目锚点】问题最终必须落到一个明确项目上；题面包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。\n2. 【可执行多跳】推理链必须锚定在同一个项目上逐步分解为对 project-query 的连续查询：先按完整项目编号定位项目 → 再查该项目的阶段/最终用户/销售工程师/成员角色。不得要求文档检索、外部知识或数值计算。\n3. 【字段锁定】只允许涉及：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n4. 【禁止反查】不得以人员、部门、组织为推理起点反查项目；人员只可作为被问内容（某项目的成员/角色）。\n5. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、分工、组织关系。\n6. 【混合片段守则】给定子图可能混有多个项目的片段。凡是无法确定归属于哪个明确项目的实体或关系，一律不用作出题素材；也不要对比/合并不同项目。\n7. 【事实可溯】答案完全由所锚定项目的子图片段推导，不得编造；答案简洁给出推导结论涉及的具体字段值。\n\n输出格式：\nquestion_text\nanswer_text\n\n给定知识子图：\n--实体--\n{entities}\n--关系--\n{relationships}\n\n输出：\n", + "zh": "你是 PMS 项目管理系统的训练题生成器。给定一个知识子图（实体与关系），生成一个需要两步及以上查询、但仍可由 project-query 技能逐步执行的中文问答对。\n\n出题铁律：\n1. 【项目锚点】问题最终必须落到一个明确项目上；题面包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。\n2. 【可执行多跳】推理链必须锚定在同一个项目上逐步分解为对 project-query 的连续查询：先按完整项目编号定位项目 → 再查该项目的阶段/最终用户/销售工程师/成员角色。不得要求文档检索、外部知识或数值计算。\n3. 【字段锁定】只允许涉及：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n4. 【禁止反查】不得以人员、部门、组织为推理起点反查项目；人员只可作为被问内容（某项目的成员/角色）。\n5. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、分工、组织关系。\n6. 【混合片段守则】给定子图可能混有多个项目的片段。凡是无法确定归属于哪个明确项目的实体或关系，一律不用作出题素材；也不要对比/合并不同项目。\n7. 【事实可溯】答案完全由所锚定项目的子图片段推导，不得编造；答案简洁给出推导结论涉及的具体字段值。\n\n输出格式：\nquestion_text\nanswer_text\n\n给定知识子图：\n--实体--\n{entities}\n--关系--\n{relationships}\n\n输出：\n" +} diff --git a/graphgen/templates/prompt_profiles.py b/graphgen/templates/prompt_profiles.py new file mode 100644 index 00000000..a825a674 --- /dev/null +++ b/graphgen/templates/prompt_profiles.py @@ -0,0 +1,119 @@ +"""Versioned prompt profiles: config-driven prompt customization for generators. + +PMS fork capability (see docs/PMS_PATCHES.md #1). Official GraphGen hard-codes all +prompts as Python constants under ``graphgen.templates``; this module adds a +generic, fail-closed override channel: + +- a *profile* is a directory containing one JSON file per generator + ``TEMPLATE_KEY`` (e.g. ``atomic.json``); +- each JSON file must mirror the exact key structure of the corresponding + official template (same languages, same nested keys, leaf values are str); +- ``profile`` values are either built-in profile ids resolved against + ``graphgen/templates/profiles/`` or explicit filesystem paths (containing + a path separator). + +Default behaviour is untouched: when no profile is applied every generator +falls back to the official template constant byte-for-byte. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from graphgen.templates import ( + AGGREGATED_GENERATION_PROMPT, + ATOMIC_GENERATION_PROMPT, + COT_GENERATION_PROMPT, + FILL_IN_BLANK_GENERATION_PROMPT, + MAQ_GENERATION_PROMPT, + MCQ_GENERATION_PROMPT, + MULTI_HOP_GENERATION_PROMPT, + TF_GENERATION_PROMPT, + VQA_GENERATION_PROMPT, +) + +BUILTIN_PROFILE_ROOT = Path(__file__).resolve().parent / "profiles" + +#: template_key -> official template constant (structure validation source). +OFFICIAL_TEMPLATES: dict[str, Any] = { + "atomic": ATOMIC_GENERATION_PROMPT, + "multi_hop": MULTI_HOP_GENERATION_PROMPT, + "aggregated": AGGREGATED_GENERATION_PROMPT, + "cot": COT_GENERATION_PROMPT, + "multi_choice": MCQ_GENERATION_PROMPT, + "multi_answer": MAQ_GENERATION_PROMPT, + "fill_in_blank": FILL_IN_BLANK_GENERATION_PROMPT, + "masked_fill_in_blank": AGGREGATED_GENERATION_PROMPT, + "true_false": TF_GENERATION_PROMPT, + "vqa": VQA_GENERATION_PROMPT, +} + + +def resolve_profile_dir(profile: str) -> Path: + """Resolve a profile id / path to its directory; fail-closed if missing.""" + if not profile or not isinstance(profile, str): + raise ValueError("prompt_profile_must_be_non_empty_string") + if "/" in profile or "\\" in profile: + profile_dir = Path(profile) + else: + profile_dir = BUILTIN_PROFILE_ROOT / profile + if not profile_dir.is_dir(): + raise ValueError(f"prompt_profile_not_found:{profile}") + return profile_dir + + +def _validate_structure(official: Any, override: Any, path: str) -> None: + """Override must mirror the official template's key structure exactly.""" + if isinstance(official, dict): + if not isinstance(override, dict): + raise ValueError(f"prompt_profile_structure_mismatch:{path}") + if set(override.keys()) != set(official.keys()): + raise ValueError( + f"prompt_profile_structure_mismatch:{path}:" + f"keys={sorted(map(str, override.keys()))}" + ) + for key in official: + _validate_structure(official[key], override[key], f"{path}.{key}") + return + if not isinstance(override, str) or not override.strip(): + raise ValueError(f"prompt_profile_structure_mismatch:{path}:non_string_leaf") + + +def load_profile_template(profile: str, template_key: str) -> dict[str, Any]: + """Load one generator's template override from a profile. + + :param profile: profile id (builtin) or filesystem path. + :param template_key: generator TEMPLATE_KEY, must exist in OFFICIAL_TEMPLATES. + :return: parsed override dict (same structure as the official template). + """ + if template_key not in OFFICIAL_TEMPLATES: + raise ValueError(f"prompt_profile_unknown_template_key:{template_key}") + profile_file = resolve_profile_dir(profile) / f"{template_key}.json" + if not profile_file.is_file(): + raise ValueError(f"prompt_profile_template_missing:{profile}/{template_key}") + try: + override = json.loads(profile_file.read_text(encoding="utf-8")) + except json.JSONDecodeError as error: + raise ValueError(f"prompt_profile_json_invalid:{profile_file}:{error}") from error + _validate_structure(OFFICIAL_TEMPLATES[template_key], override, template_key) + return override + + +def profile_template_sha256s(profile: str, template_keys: list[str]) -> dict[str, str]: + """Lineage fingerprint: SHA-256 of each override file actually in use.""" + profile_dir = resolve_profile_dir(profile) + digests: dict[str, str] = {} + for template_key in template_keys: + profile_file = profile_dir / f"{template_key}.json" + if not profile_file.is_file(): + continue + digests[template_key] = hashlib.sha256(profile_file.read_bytes()).hexdigest() + return digests + + +def apply_prompt_profile(generator: Any, profile: str) -> None: + """Attach profile templates to a generator instance (BaseGenerator API).""" + generator.apply_prompt_profile(profile) From 2adbbe4c52f9776dc3538fc3f954bb19707cb0f5 Mon Sep 17 00:00:00 2001 From: Leon-L20 Date: Fri, 9 Oct 2026 09:26:27 +0000 Subject: [PATCH 2/9] =?UTF-8?q?feat(pms):=20Phase2=20=E4=BA=8B=E4=BB=B6-?= =?UTF-8?q?=E5=AE=9E=E4=BD=93=E6=98=9F=E5=9E=8B=E5=9B=BE=E7=AE=A1=E7=BA=BF?= =?UTF-8?q?=EF=BC=88SAG=20=E7=A7=BB=E6=A4=8D=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 事件抽取：SAG extract_document v3.1 提示词改编 + 严格响应合同 + 数字接地校验（MIT 归因） - 星型编码进图：事件=EVENT 节点+成员边（边属性含角色），merge/SEP 血缘复用官方 LightRAG - build_kg kg_method 分发：event_entity 与 light_rag 并存，默认行为逐字节不变 - event_join 分区器：共享实体 join + 同锚硬约束，产出官方 Community，metadata 带 event_ids - 审计链：生成端引用分区节点集合校验，失败剥离保留题面；pms_policy.v3 profile - 台账 docs/PMS_PATCHES.md #7；fork 测试基线与上游一致（23F/6P，零回归） Co-Authored-By: Claude Opus 5.5 --- docs/PMS_PATCHES.md | 18 +++ graphgen/bases/base_generator.py | 7 +- graphgen/models/__init__.py | 2 + graphgen/models/generator/atomic_generator.py | 7 +- .../models/generator/multi_hop_generator.py | 7 +- graphgen/models/generator/support.py | 42 ++++++ graphgen/models/kg_builder/event_contract.py | 119 +++++++++++++++ .../kg_builder/event_entity_kg_builder.py | 135 ++++++++++++++++++ graphgen/models/partitioner/__init__.py | 1 + .../partitioner/event_join_partitioner.py | 113 +++++++++++++++ graphgen/operators/build_kg/build_event_kg.py | 51 +++++++ .../operators/build_kg/build_kg_service.py | 13 ++ .../operators/generate/generate_service.py | 15 +- .../operators/partition/partition_service.py | 8 ++ graphgen/templates/__init__.py | 7 +- graphgen/templates/kg/__init__.py | 1 + .../templates/kg/event_entity_extraction.py | 113 +++++++++++++++ .../profiles/pms_policy.v3/atomic.json | 4 + .../profiles/pms_policy.v3/multi_hop.json | 4 + 19 files changed, 659 insertions(+), 8 deletions(-) create mode 100644 graphgen/models/generator/support.py create mode 100644 graphgen/models/kg_builder/event_contract.py create mode 100644 graphgen/models/kg_builder/event_entity_kg_builder.py create mode 100644 graphgen/models/partitioner/event_join_partitioner.py create mode 100644 graphgen/operators/build_kg/build_event_kg.py create mode 100644 graphgen/templates/kg/event_entity_extraction.py create mode 100644 graphgen/templates/profiles/pms_policy.v3/atomic.json create mode 100644 graphgen/templates/profiles/pms_policy.v3/multi_hop.json diff --git a/docs/PMS_PATCHES.md b/docs/PMS_PATCHES.md index 0d1ee4a9..88d2fb8e 100644 --- a/docs/PMS_PATCHES.md +++ b/docs/PMS_PATCHES.md @@ -55,6 +55,24 @@ 时对 logprob 调用注入 `extra_body.chat_template_kwargs.enable_thinking=false` （Qwen 兼容后端）。普通生成调用不受影响。 +## #7 事件-实体星型图管线（Phase 2，2026-10-09） + +SAG 思想移植（MIT，arXiv:2606.15971；参考源码 `workspace/sag_reference/`，方案见 PMS 主仓 +`docs/2026-09-18-graphgen-phase2-sag-event-entity-plan.md` §3.2 复用地图）： + +- `templates/kg/event_entity_extraction.py`：事件抽取提示词（SAG extract_document v3.1 改编 + + PMS 锚点/字段锁定/数字接地铁律）。 +- `models/kg_builder/event_contract.py`：纯 stdlib 响应合同（禁合同外字段、is_valid 自报、 + 无锚 fail-closed 丢弃并计数）+ 数字接地校验（SAG grounding 移植）+ 确定性事件 ID。 +- `models/kg_builder/event_entity_kg_builder.py`：事件→星型编码（EVENT 节点 + 成员边， + 边属性含角色）；合并/摘要/SEP 血缘复用官方 LightRAG merge。 +- `operators/build_kg/`：`kg_method: event_entity` 分发（默认 light_rag 与官方逐字节一致）。 +- `models/partitioner/event_join_partitioner.py`：共享实体 join + 同锚硬约束的事件簇分区 + （SAG join 语义离线物化），产出官方 `Community`，metadata 携带 event_ids。 +- `models/generator/support.py` + atomic/multi_hop 解析 + GenerateService 校验： + `{"cited": [...]}` 引用必须存在于本分区节点集合，失败剥离保留题面。 +- `templates/profiles/pms_policy.v3/`：v2 模板 + support 输出要求。 + ## 验收 - 默认路径（无 profile/gate/并发参数）与官方行为逐字节一致； diff --git a/graphgen/bases/base_generator.py b/graphgen/bases/base_generator.py index 04867b88..9307dea0 100644 --- a/graphgen/bases/base_generator.py +++ b/graphgen/bases/base_generator.py @@ -84,6 +84,8 @@ def format_generation_results( ) -> dict[str, Any]: question = result.get("question", "") answer = result.get("answer", "") + # PMS fork（Phase 2）：已校验的 引用块随 QA_pairs 输出透传。 + support = result.get("support") if "options" in result and result["options"]: options = result["options"] options_str = "\n".join( @@ -114,8 +116,11 @@ def format_generation_results( } if output_data_format == "QA_pairs": - return { + output = { "question": question, "answer": answer, } + if support: + output["support"] = support + return output raise ValueError(f"Unknown output data format: {output_data_format}") diff --git a/graphgen/models/__init__.py b/graphgen/models/__init__.py index 8bd2c9d8..d6c9306e 100644 --- a/graphgen/models/__init__.py +++ b/graphgen/models/__init__.py @@ -30,6 +30,7 @@ BFSPartitioner, DFSPartitioner, ECEPartitioner, + EventJoinPartitioner, LeidenPartitioner, QuintuplePartitioner, TriplePartitioner, @@ -89,6 +90,7 @@ "BFSPartitioner": ".partitioner", "DFSPartitioner": ".partitioner", "ECEPartitioner": ".partitioner", + "EventJoinPartitioner": ".partitioner", "LeidenPartitioner": ".partitioner", "TriplePartitioner": ".partitioner", "QuintuplePartitioner": ".partitioner", diff --git a/graphgen/models/generator/atomic_generator.py b/graphgen/models/generator/atomic_generator.py index a77b731c..38fe9d98 100644 --- a/graphgen/models/generator/atomic_generator.py +++ b/graphgen/models/generator/atomic_generator.py @@ -2,6 +2,7 @@ from typing import Any from graphgen.bases import BaseGenerator +from graphgen.models.generator.support import parse_support from graphgen.utils import detect_main_language, logger @@ -45,4 +46,8 @@ def parse_response(response: str) -> list[dict]: answer = answer.strip('"').strip("'") logger.debug("Question: %s", question) logger.debug("Answer: %s", answer) - return [{"question": question, "answer": answer}] + qa_pairs = [{"question": question, "answer": answer}] + support = parse_support(response) + if support: + qa_pairs[0]["support"] = support + return qa_pairs diff --git a/graphgen/models/generator/multi_hop_generator.py b/graphgen/models/generator/multi_hop_generator.py index 232f0e56..da0a4615 100644 --- a/graphgen/models/generator/multi_hop_generator.py +++ b/graphgen/models/generator/multi_hop_generator.py @@ -2,6 +2,7 @@ from typing import Any from graphgen.bases import BaseGenerator +from graphgen.models.generator.support import parse_support from graphgen.utils import detect_main_language, logger @@ -48,4 +49,8 @@ def parse_response(response: str) -> list[dict]: answer = answer.strip('"').strip("'") logger.debug("Question: %s", question) logger.debug("Answer: %s", answer) - return [{"question": question, "answer": answer}] + qa_pairs = [{"question": question, "answer": answer}] + support = parse_support(response) + if support: + qa_pairs[0]["support"] = support + return qa_pairs diff --git a/graphgen/models/generator/support.py b/graphgen/models/generator/support.py new file mode 100644 index 00000000..810f65d2 --- /dev/null +++ b/graphgen/models/generator/support.py @@ -0,0 +1,42 @@ +""" 审计块解析（PMS fork，Phase 2 升级 C）。 + +生成器可在 / 之后输出 +``{"cited": ["节点或事件名称", ...]}``； +本模块只负责解析，有效性校验（cited 是否真存在于出题分区）在 +GenerateService.process 中按分区的节点集合执行，失败剥离 support 保留题面。 +""" + +from __future__ import annotations + +import json +import re +from typing import Any, Optional + +_SUPPORT_RE = re.compile(r"(.*?)", re.DOTALL) + + +def parse_support(response: str) -> Optional[dict[str, Any]]: + """从模型响应解析 support 块；缺失/畸形返回 None（不影响题面解析）。""" + match = _SUPPORT_RE.search(response or "") + if not match: + return None + try: + payload = json.loads(match.group(1).strip()) + except json.JSONDecodeError: + return None + if not isinstance(payload, dict) or set(payload.keys()) != {"cited"}: + return None + cited = payload.get("cited") + if not isinstance(cited, list) or not all(isinstance(c, str) and c.strip() for c in cited): + return None + return {"cited": [c.strip() for c in cited]} + + +def validate_support(support: Optional[dict[str, Any]], node_names: set[str]) -> Optional[dict[str, Any]]: + """cited 全部存在于分区节点集合 → 原样返回；否则返回 None（剥离）。""" + if not support: + return None + cited = support.get("cited") or [] + if cited and set(cited) <= node_names: + return support + return None diff --git a/graphgen/models/kg_builder/event_contract.py b/graphgen/models/kg_builder/event_contract.py new file mode 100644 index 00000000..374ea557 --- /dev/null +++ b/graphgen/models/kg_builder/event_contract.py @@ -0,0 +1,119 @@ +"""Pure stdlib contracts for SAG-adapted event extraction (testable without GraphGen).""" +from __future__ import annotations +import hashlib +import json +import re +from collections import defaultdict +from typing import Any, Optional + +PMS_ENTITY_TYPES: list[dict] = [ + {"type": "PROJECT", "description": "项目本体，含完整项目编号"}, + {"type": "PERSON", "description": "人员，项目成员或干系人"}, + {"type": "ROLE", "description": "项目内职务或角色"}, + {"type": "ORGANIZATION", "description": "组织、部门、公司或团队"}, + {"type": "PHASE", "description": "项目阶段或状态"}, + {"type": "LOCATION", "description": "地点或区域"}, + {"type": "DATE", "description": "日期或时间"}, + {"type": "METRIC", "description": "数量、比例等指标"}, +] + +PROJECT_ID_RE = re.compile(r"\b[PI]-\d{6,}\b") +_NUMBER_RE = re.compile(r"(? set[str]: + """原文/事件中的数字集合（SAG grounding 的确定性口径）。""" + return set(_NUMBER_RE.findall(text or "")) + + +def event_anchor(content: str) -> str | None: + """事件的完整项目编号锚点；取首个出现的编号。""" + match = PROJECT_ID_RE.search(content or "") + return match.group(0) if match else None + + +def event_node_name(anchor: str, title: str, content: str) -> str: + """确定性事件节点 id：event:{anchor}:{hash12}（内容 SHA，重叠 chunk 天然去重）。""" + digest = hashlib.sha256(f"{anchor}\n{title}\n{content}".encode("utf-8")).hexdigest()[:12] + return f"event:{anchor}:{digest}" + + +def parse_extraction_response(response: str, *, chunk_content: str) -> tuple[list[dict], dict[str, int]]: + """解析并校验 LLM 事件抽取响应，返回 (有效事件列表, 丢弃计数)。 + + 严格合同（SAG schema.py 口径）：顶层/事件/实体禁合同外字段；is_valid=False、 + 无项目锚点、数字不接地、实体类型越界的事件整条丢弃并计数。 + """ + dropped: dict[str, int] = defaultdict(int) + text = (response or "").strip() + if text.startswith("```"): + text = text.strip("`") + if text.startswith("json"): + text = text[4:] + text = text.strip() + try: + payload = json.loads(text) + except json.JSONDecodeError: + dropped["response_not_json"] += 1 + return [], dict(dropped) + if not isinstance(payload, dict) or set(payload.keys()) != {"type", "data"}: + dropped["response_contract_invalid"] += 1 + return [], dict(dropped) + data = payload.get("data") + if not isinstance(data, dict) or set(data.keys()) != {"items"}: + dropped["response_contract_invalid"] += 1 + return [], dict(dropped) + + evidence_numbers = numbers_in(chunk_content) + valid_events: list[dict] = [] + for item in data.get("items") or []: + if not isinstance(item, dict): + dropped["event_not_object"] += 1 + continue + extra_keys = set(item.keys()) - _ALLOWED_EVENT_KEYS + if extra_keys: + dropped["event_extra_fields"] += 1 + continue + title = str(item.get("title") or "").strip() + content = str(item.get("content") or "").strip() + entities_raw = item.get("entities") or [] + if not title or not content: + dropped["event_blank"] += 1 + continue + if item.get("is_valid") is False: + dropped["event_self_invalid"] += 1 + continue + anchor = event_anchor(content) + if anchor is None: + dropped["event_missing_anchor"] += 1 + continue + ungrounded = {n for n in numbers_in(f"{title} {content}") if n not in evidence_numbers} + if ungrounded: + dropped["event_number_not_grounded"] += 1 + continue + entities: list[dict] = [] + for entity in entities_raw: + if not isinstance(entity, dict): + dropped["entity_not_object"] += 1 + continue + if set(entity.keys()) - _ALLOWED_ENTITY_KEYS: + dropped["entity_extra_fields"] += 1 + continue + etype = str(entity.get("type") or "").strip().upper() + ename = str(entity.get("name") or "").strip() + edesc = str(entity.get("description") or "").strip() + if not etype or not ename or not edesc: + dropped["entity_blank"] += 1 + continue + if etype not in _ALLOWED_TYPES: + dropped["entity_type_unknown"] += 1 + continue + entities.append({"type": etype, "name": ename, "description": edesc}) + valid_events.append( + {"title": title, "content": content, "anchor": anchor, "entities": entities} + ) + return valid_events, dict(dropped) + diff --git a/graphgen/models/kg_builder/event_entity_kg_builder.py b/graphgen/models/kg_builder/event_entity_kg_builder.py new file mode 100644 index 00000000..f247cde0 --- /dev/null +++ b/graphgen/models/kg_builder/event_entity_kg_builder.py @@ -0,0 +1,135 @@ +"""Event-entity KG builder (PMS fork, SAG-adapted). + +来源与许可：核心设计改编自 Zleap-AI/SAG `zleap/sag/modules/extract/{schema,grounding}.py` +与 `prompts/extract_document.yaml` v3.1（MIT，Copyright (c) 2026 sag contributors， +arXiv:2606.15971）。改编点见 PMS fork docs/PMS_PATCHES.md #7 与 Phase 2 方案文档 §3.2。 + +与 LightRAGKGBuilder 的差异： +- 每 chunk 抽取"语义完整的事件"而非三元组：事件 = EVENT 节点 + "事件→参与者" + 星型边（超边的星型编码，n 元关系不拆三元组）； +- 严格响应合同（禁合同外字段、is_valid 自报、无锚 fail-closed 丢弃）； +- 数字接地（SAG grounding 移植）：事件中一切数字必须在原文逐字出现，否则弃； +- 事件节点 id 确定性（内容 SHA），chunk 重叠去重由官方 merge_nodes SEP 合并免费获得。 + +extract() 返回与官方 ({entity_name: [node]}, {(src, tgt): [edge]}) 相同的形状， +merge_nodes / merge_edges / 描述摘要全部复用官方实现。 +""" + +from __future__ import annotations + +import json + +from collections import defaultdict +from typing import Any, Dict, List, Tuple + +from graphgen.bases import BaseGraphStorage, BaseLLMWrapper, Chunk +from graphgen.templates import EVENT_ENTITY_EXTRACTION_PROMPT +from graphgen.templates.kg.event_entity_extraction import PMS_ENTITY_TYPES +from graphgen.utils import detect_main_language, logger + +from graphgen.models.kg_builder.event_contract import ( + PROJECT_ID_RE, + _ALLOWED_ENTITY_KEYS, + _ALLOWED_EVENT_KEYS, + _ALLOWED_TYPES, + event_anchor, + event_node_name, + numbers_in, + parse_extraction_response, +) + + +class EventEntityKGBuilder: + """事件-实体抽取器：extract → 星型编码节点/边；合并复用官方 LightRAG merge。""" + + def __init__(self, llm_client: BaseLLMWrapper): + from graphgen.models import LightRAGKGBuilder + + self.llm_client = llm_client + self._merger = LightRAGKGBuilder(llm_client=llm_client) + + async def extract( + self, chunk: Chunk + ) -> Tuple[Dict[str, List[dict]], Dict[Tuple[str, str], List[dict]]]: + """单 chunk 事件抽取，返回与官方同形的 (nodes_data, edges_data)。""" + content = chunk.content + language = detect_main_language(content) + entity_types_json = json.dumps(PMS_ENTITY_TYPES, ensure_ascii=False) + request_json = json.dumps( + { + "type": "request", + "data": { + "items": [{"id": 1, "content": content}], + "meta": {"entity_types": "__ENTITY_TYPES__"}, + }, + }, + ensure_ascii=False, + ).replace("__ENTITY_TYPES__", entity_types_json) + # 模板含 JSON 字面大括号，禁用 str.format；用占位符替换注入输入。 + prompt = EVENT_ENTITY_EXTRACTION_PROMPT[language].replace("{input_text}", request_json) + + response = await self.llm_client.generate_answer(prompt) + events, dropped = parse_extraction_response(response, chunk_content=content) + if sum(dropped.values()): + logger.warning( + "Event extraction dropped: %s (chunk=%s)", dropped, chunk.id + ) + if not events: + return {}, {} + + nodes: Dict[str, List[dict]] = defaultdict(list) + edges: Dict[Tuple[str, str], List[dict]] = defaultdict(list) + for event in events: + anchor = event["anchor"] + event_name = event_node_name(anchor, event["title"], event["content"]) + description = f"{event['title']}：{event['content']}" + # 事件节点（EVENT 实体，扁平属性；参与者由星型边表达） + nodes[event_name].append( + { + "entity_type": "EVENT", + "entity_name": event_name, + "description": description, + "source_id": chunk.id, + } + ) + # 参与者实体（去重保序；锚点项目实体若缺失则补齐） + seen: set[str] = set() + participants: list[dict] = [] + for entity in event["entities"]: + if entity["name"] not in seen: + seen.add(entity["name"]) + participants.append(entity) + if anchor not in seen: + participants.append( + { + "type": "PROJECT", + "name": anchor, + "description": f"项目 {anchor}，本事项的锚定项目", + } + ) + seen.add(anchor) + for entity in participants: + nodes[entity["name"]].append( + { + "entity_type": entity["type"], + "entity_name": entity["name"], + "description": entity["description"], + "source_id": chunk.id, + } + ) + edges[(event_name, entity["name"])].append( + { + "src_id": event_name, + "tgt_id": entity["name"], + "description": f"事件参与者：{entity['name']}（{entity['type']}）—— {entity['description']}", + "source_id": chunk.id, + "role": entity["type"], + } + ) + return dict(nodes), dict(edges) + + async def merge_nodes(self, node_data: tuple, kg_instance: BaseGraphStorage) -> dict: + return await self._merger.merge_nodes(node_data, kg_instance=kg_instance) + + async def merge_edges(self, edges_data: tuple, kg_instance: BaseGraphStorage) -> dict: + return await self._merger.merge_edges(edges_data, kg_instance=kg_instance) diff --git a/graphgen/models/partitioner/__init__.py b/graphgen/models/partitioner/__init__.py index 4306f247..8d5b20bb 100644 --- a/graphgen/models/partitioner/__init__.py +++ b/graphgen/models/partitioner/__init__.py @@ -2,6 +2,7 @@ from .bfs_partitioner import BFSPartitioner from .dfs_partitioner import DFSPartitioner from .ece_partitioner import ECEPartitioner +from .event_join_partitioner import EventJoinPartitioner from .leiden_partitioner import LeidenPartitioner from .quintuple_partitioner import QuintuplePartitioner from .triple_partitioner import TriplePartitioner diff --git a/graphgen/models/partitioner/event_join_partitioner.py b/graphgen/models/partitioner/event_join_partitioner.py new file mode 100644 index 00000000..48d51b7b --- /dev/null +++ b/graphgen/models/partitioner/event_join_partitioner.py @@ -0,0 +1,113 @@ +"""Event-join partitioner (PMS fork, SAG join semantics). + +来源：共享实体 join 语义翻译自 Zleap-AI/SAG `modules/search/base.py` 的 +entity→event→relation 两跳查询（MIT，arXiv:2606.15971）；离线物化见 Phase 2 方案文档。 + +事件 = EVENT 节点（id 形如 ``event:{anchor}:{hash12}``）+ 星型成员边。分区算法： +1. 种子事件按节点 id 排序（确定性）； +2. 通过共享参与者实体 join 一跳邻居事件； +3. 硬约束：只接受同 project_anchor 事件（簇项目纯度 100%）； +4. 容量上限：max_events_per_community 或事件描述 token 总和。 + +产出官方 ``Community``（nodes = 事件+参与者实体，edges = 星型成员边）， +metadata 携带 event_ids 供出题端 support 校验。 +""" + +import math +from collections import deque +from typing import Any, Iterable, List, Optional, Set, Tuple + +from graphgen.bases import BaseGraphStorage +from graphgen.bases.datatypes import Community + +EVENT_NODE_PREFIX = "event:" + + +class EventJoinPartitioner: + """EVENT 星型子图 → 同项目事件簇（官方 Community 数据类型）。""" + + def partition( + self, + g: BaseGraphStorage, + max_events_per_community: int = 6, + max_tokens_per_community: int = 4096, + min_events_per_community: int = 1, + **kwargs: Any, + ) -> Iterable[Community]: + events: List[Tuple[str, dict]] = [] + node_dict: dict[str, dict] = {} + for nid, data in g.get_all_nodes(): + node_dict[nid] = data + if str(data.get("entity_type", "")).upper() == "EVENT": + events.append((nid, data)) + events.sort(key=lambda item: item[0]) + + # 参与者 → 事件 倒排索引（共享实体 join key） + events_by_entity: dict[str, List[str]] = {} + star_edges: dict[str, List[str]] = {} + for event_id, _ in events: + star_edges[event_id] = [] + for neighbor in g.get_neighbors(event_id): + star_edges[event_id].append(neighbor) + events_by_entity.setdefault(neighbor, []).append(event_id) + + used: Set[str] = set() + + def _anchor(event_id: str) -> str: + # event:{anchor}:{hash} + return event_id[len(EVENT_NODE_PREFIX):].rsplit(":", 1)[0] + + def _grow(seed: Tuple[str, dict]) -> Optional[Community]: + member_events: List[str] = [] + event_set: Set[str] = set() + member_entities: dict[str, dict] = {} + member_edges: Set[frozenset] = set() + token_sum = 0 + anchor = _anchor(seed[0]) + queue = deque([seed[0]]) + + while queue: + event_id = queue.popleft() + if event_id in event_set: + continue + event_data = node_dict[event_id] + if token_sum + int(event_data.get("length", 0)) > max_tokens_per_community: + continue + event_set.add(event_id) + member_events.append(event_id) + token_sum += int(event_data.get("length", 0)) + for entity_id in star_edges.get(event_id, []): + if entity_id not in member_entities: + entity_data = node_dict.get(entity_id) + if entity_data is not None: + member_entities[entity_id] = entity_data + edge_key = frozenset((event_id, entity_id)) + member_edges.add(edge_key) + # 共享实体 join 一跳扩展；跨锚事件被硬约束拒绝 + for other in events_by_entity.get(entity_id, []): + if other in used or other in event_set: + continue + if _anchor(other) != anchor: + continue + if token_sum + int(node_dict[other].get("length", 0)) > max_tokens_per_community: + continue + queue.append(other) + if len(event_set) >= max_events_per_community: + break + + if len(member_events) < min_events_per_community: + return None + used.update(event_set) + return Community( + id=seed[0], + nodes=member_events + sorted(member_entities), + edges=[tuple(sorted(edge)) for edge in sorted(member_edges, key=sorted)], + metadata={"event_ids": member_events, "anchor": anchor}, + ) + + for event_id, data in events: + if event_id in used: + continue + community = _grow((event_id, data)) + if community: + yield community diff --git a/graphgen/operators/build_kg/build_event_kg.py b/graphgen/operators/build_kg/build_event_kg.py new file mode 100644 index 00000000..a25ca139 --- /dev/null +++ b/graphgen/operators/build_kg/build_event_kg.py @@ -0,0 +1,51 @@ +"""Event-entity KG construction over text chunks (PMS fork).""" + +from __future__ import annotations + +from collections import defaultdict +from typing import List + +from graphgen.bases import BaseLLMWrapper +from graphgen.bases.base_storage import BaseGraphStorage +from graphgen.bases.datatypes import Chunk +from graphgen.models.kg_builder.event_entity_kg_builder import EventEntityKGBuilder +from graphgen.utils import run_concurrent + + +def build_event_entity_kg( + llm_client: BaseLLMWrapper, + kg_instance: BaseGraphStorage, + chunks: List[Chunk], +) -> tuple: + """chunks → 事件星型图（合并/摘要/SEP 血缘全部复用官方 LightRAG merge）。""" + kg_builder = EventEntityKGBuilder(llm_client=llm_client) + + results = run_concurrent( + kg_builder.extract, + chunks, + desc="[2/4]Extracting events and entities from chunks", + unit="chunk", + ) + results = [res for res in results if res] + + nodes = defaultdict(list) + edges = defaultdict(list) + for n, e in results: + for k, v in n.items(): + nodes[k].extend(v) + for k, v in e.items(): + edges[tuple(sorted(k))].extend(v) + + nodes = run_concurrent( + lambda kv: kg_builder.merge_nodes(kv, kg_instance=kg_instance), + list(nodes.items()), + desc="Inserting events and entities into storage", + ) + + edges = run_concurrent( + lambda kv: kg_builder.merge_edges(kv, kg_instance=kg_instance), + list(edges.items()), + desc="Inserting event membership into storage", + ) + + return nodes, edges diff --git a/graphgen/operators/build_kg/build_kg_service.py b/graphgen/operators/build_kg/build_kg_service.py index 19fa9e7a..9a3c484c 100644 --- a/graphgen/operators/build_kg/build_kg_service.py +++ b/graphgen/operators/build_kg/build_kg_service.py @@ -6,6 +6,7 @@ from graphgen.common.init_storage import init_storage from graphgen.utils import logger +from .build_event_kg import build_event_entity_kg from .build_mm_kg import build_mm_kg from .build_text_kg import build_text_kg @@ -27,6 +28,9 @@ def __init__( ) self.build_kwargs = build_kwargs self.max_loop: int = int(self.build_kwargs.get("max_loop", 3)) + # PMS fork（Phase 2）：文本 KG 抽取方法分发。默认 light_rag 与官方逐字节 + # 一致；event_entity 走事件-实体星型编码（见 models/kg_builder/event_entity_kg_builder.py）。 + self.kg_method: str = str(self.build_kwargs.get("kg_method", "light_rag")) def process(self, batch: list) -> Tuple[list, dict]: """ @@ -49,6 +53,15 @@ def process(self, batch: list) -> Tuple[list, dict]: if len(text_chunks) == 0: logger.info("All text chunks are already in the storage") + elif self.kg_method == "event_entity": + logger.info("[Text Event and Entity Extraction] processing ...") + text_nodes, text_edges = build_event_entity_kg( + llm_client=self.llm_client, + kg_instance=self.graph_storage, + chunks=text_chunks, + ) + nodes += text_nodes + edges += text_edges else: logger.info("[Text Entity and Relation Extraction] processing ...") text_nodes, text_edges = build_text_kg( diff --git a/graphgen/operators/generate/generate_service.py b/graphgen/operators/generate/generate_service.py index 52359b29..2179a1eb 100644 --- a/graphgen/operators/generate/generate_service.py +++ b/graphgen/operators/generate/generate_service.py @@ -130,18 +130,25 @@ def process(self, batch: list) -> Tuple[list, dict]: unit="batch", ) + from graphgen.models.generator.support import validate_support + meta_updates = {} final_results = [] - for input_trace_id, qa_pairs in zip( - [item["_trace_id"] for item in batch], results - ): + for item, qa_pairs in zip(batch, results): if not qa_pairs: continue + # PMS fork（Phase 2 升级 C）：.cited 必须全部存在于本分区 + # 节点集合，否则整块剥离、题面保留（题面质量与审计质量解耦）。 + node_names = {node[0] for node in item.get("nodes", []) if isinstance(node, (list, tuple)) and node} for qa_pair in qa_pairs: + support = qa_pair.pop("support", None) if isinstance(qa_pair, dict) else None + support = validate_support(support, node_names) + if support: + qa_pair["support"] = support res = self.generator.format_generation_results( qa_pair, output_data_format=self.data_format ) res["_trace_id"] = self.get_trace_id(res) final_results.append(res) - meta_updates.setdefault(input_trace_id, []).append(res["_trace_id"]) + meta_updates.setdefault(item["_trace_id"], []).append(res["_trace_id"]) return final_results, meta_updates diff --git a/graphgen/operators/partition/partition_service.py b/graphgen/operators/partition/partition_service.py index b55c5159..6e7ddff3 100644 --- a/graphgen/operators/partition/partition_service.py +++ b/graphgen/operators/partition/partition_service.py @@ -38,6 +38,10 @@ def __init__( from graphgen.models import DFSPartitioner self.partitioner = DFSPartitioner() + elif method == "event_join": + from graphgen.models import EventJoinPartitioner + + self.partitioner = EventJoinPartitioner() elif method == "ece": # before ECE partitioning, we need to: # 'quiz' and 'judge' to get the comprehension loss if unit_sampling is not random @@ -87,6 +91,10 @@ def generator(): "nodes": b[0], "edges": b[1], } + # PMS fork（Phase 2）：事件簇附带 event_ids，供出题端 support 校验。 + event_ids = list(getattr(community, "metadata", {}) or {}).get("event_ids") or [] + if event_ids: + result["events"] = event_ids result["_trace_id"] = self.get_trace_id(result) yield result logger.info("Total communities partitioned: %d", count) diff --git a/graphgen/templates/__init__.py b/graphgen/templates/__init__.py index 0b6a865d..dd2cb60d 100644 --- a/graphgen/templates/__init__.py +++ b/graphgen/templates/__init__.py @@ -13,7 +13,12 @@ TF_GENERATION_PROMPT, VQA_GENERATION_PROMPT, ) -from .kg import KG_EXTRACTION_PROMPT, KG_SUMMARIZATION_PROMPT, MMKG_EXTRACTION_PROMPT +from .kg import ( + EVENT_ENTITY_EXTRACTION_PROMPT, + KG_EXTRACTION_PROMPT, + KG_SUMMARIZATION_PROMPT, + MMKG_EXTRACTION_PROMPT, +) from .rephrasing import STYLE_CONTROLLED_REPHRASING_PROMPTS from .search_judgement import SEARCH_JUDGEMENT_PROMPT from .statement_judgement import STATEMENT_JUDGEMENT_PROMPT diff --git a/graphgen/templates/kg/__init__.py b/graphgen/templates/kg/__init__.py index ea865ce6..44c48f5e 100644 --- a/graphgen/templates/kg/__init__.py +++ b/graphgen/templates/kg/__init__.py @@ -1,3 +1,4 @@ +from .event_entity_extraction import EVENT_ENTITY_EXTRACTION_PROMPT from .kg_extraction import KG_EXTRACTION_PROMPT from .kg_summarization import KG_SUMMARIZATION_PROMPT from .mm_kg_extraction import MMKG_EXTRACTION_PROMPT diff --git a/graphgen/templates/kg/event_entity_extraction.py b/graphgen/templates/kg/event_entity_extraction.py new file mode 100644 index 00000000..e74c1be8 --- /dev/null +++ b/graphgen/templates/kg/event_entity_extraction.py @@ -0,0 +1,113 @@ +# pylint: disable=C0301 +"""事件-实体抽取提示词（SAG extract_document v3.1 改编，MIT）。 + +来源：Zleap-AI/SAG `zleap/sag/prompts/extract_document.yaml`（arXiv 2606.15971）。 +改编点（PMS project-query 场景）： +- 事件必须含完整项目编号锚点（P-/I- + ≥6 位数字），无锚事件直接不产出； +- 字段锁定 / 反查禁止 / 越界语义禁止（与 pms_policy 出题铁律同源）; +- 数字接地：事件中一切数字必须在原文出现； +- 只输出扁平事项（不用 children 层级），便于星型编码进图。 +响应是严格 JSON 合同：禁合同外字段，无效字段/无锚/数字不接地一律丢弃并计数。 +""" + +TEMPLATE_ZH: str = """## Role + +你是专业的 PMS 项目管理文档事项与实体提取器。输入是当前 Chunk 片段。你的输出用于 +构建可追溯的事项—实体图谱，供后续生成"项目查询"问答题使用。 + +## 事项提取原则 + +- 按主题组织信息，用完整的主谓宾句陈述事实，禁止机械拼接原文。 +- 一个事项应围绕一个清晰主题；content 应概括"发生了什么、涉及谁、结果如何"， + 保留必要的数字和结论。 +- 可以调整语序、消解有充分依据的指代，但不得改变主语、动作性质、因果或完成状态。 +- "计划、预计、建议、可能、正在、已经完成"必须严格区分。 +- 不得把外部常识、文件名或模型推测写成原文事实。 +- 每个事项必须锚定在一个明确项目上：content 中必须出现该项目的完整项目编号 + （形如 P-24084947、I-26015101，字母+横线+至少 6 位数字）。无法锚定到完整项目 + 编号的事项不得输出。 +- 项目名称在同系统可能不唯一：提及时必须连同完整项目编号一起出现。 +- 字段锁定：只允许陈述项目编号、项目名称、项目阶段、最终用户、销售工程师、 + 项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件信息不提取。 +- 禁止反查表述：人员只作为"某项目的成员/某项目某角色由谁担任"陈述， + 不得写成"某人属于哪个部门"等组织反查事实。 + +## 实体提取原则 + +- 只提取理解事项所必需的实体；实体类型只能从输入 meta.entity_types 中选择， + 没有合适类型时不要自造类型。 +- 并列实体必须拆分，例如"甲公司和乙公司"应为两个 organization 实体。 +- description 必须描述实体在本事项中的角色、动作或关系，不能只重复实体名称。 +- 代词只有在当前片段能唯一消歧时才替换；无法确定时保留原文表达。 +- 同名实体保持一致；不同实体不得因名称相近而合并。 + +## 事实边界（fail-closed） + +- 事项的每个事实、数字、结论都必须在当前 Chunk 中有直接依据。 +- 事项 content 中出现的所有数字（含编号中的数字）必须能在原文中逐字找到； + 找不到就删掉该表述或放弃该事项。 +- 当前片段信息不足以支撑独立事项时必须返回空 items，不能靠推测拼出事项。 + +## Task + +按以下顺序执行，不能跳步： + +1. **全局扫描**：识别主体、动作、对象、时间、地点、数据、结果与不确定性； + 页眉页脚、目录编号、广告、版权声明、乱码视为噪音。 +2. **聚合或拆分判断**：优先级固定为 总结性内容聚合 > 总结后的详细展开分离 > + 普通内容按主题拆分；仅有名称或修辞差异不强行拆分。 +3. **事项构建**：title 精炼表达主题与关键结论；content 独立可读、含完整项目 + 编号锚点、禁止推理说明。 +4. **实体提取**：只从该事项陈述中提取实体，保留原文正式名称、数值单位和时间说法。 +5. **最终自检**：确认无原文之外的事实；确认每个事项含完整项目编号锚点； + 确认所有数字可在原文找到；确认输出只有统一合同允许的字段。 + 全部片段为噪音或信息不足时返回 items: []。 + +## Input + +输入 JSON：{"type": "request", "data": {"items": [{"id": 1, "content": "..."}], +"meta": {"entity_types": [{"type": "...", "description": "..."}]}}} + +## Output requirements + +- 只返回统一 JSON 合同，不要 Markdown、解释文字或合同外字段： + {"type": "response", "data": {"items": [ + {"title": "...", "content": "...", + "entities": [{"type": "...", "name": "...", "description": "..."}], + "is_valid": true} + ]}} +- 无有效事项时返回 {"type": "response", "data": {"items": []}}，不得返回 null + 或制造占位事项。 +- 相对时间（如"下月"）保留原文，不自行换算。 + +给定输入： +{input_text} + +输出： +""" + +#: 紧凑 JSON 请求包装（与 SAG request 合同同形；PMS 单 chunk 单 items） +INPUT_FORMAT: dict = { + "items_template": '{{"type": "request", "data": {{"items": [{{"id": 1, "content": {content!r}}}], ' + '"meta": {{"entity_types": {entity_types}}}}}}}', +} + +#: PMS 实体类型集（SAG 5W1H 类型集收敛到 project-query 可查询维度） +PMS_ENTITY_TYPES: list[dict] = [ + {"type": "PROJECT", "description": "项目本体，含完整项目编号"}, + {"type": "PERSON", "description": "人员，项目成员或干系人"}, + {"type": "ROLE", "description": "项目内职务或角色，如二级项目经理、销售工程师"}, + {"type": "ORGANIZATION", "description": "组织、部门、公司或团队"}, + {"type": "PHASE", "description": "项目阶段或状态，如售前、执行"}, + {"type": "LOCATION", "description": "地点或区域"}, + {"type": "DATE", "description": "日期或时间"}, + {"type": "METRIC", "description": "数量、比例等指标"}, +] + +EVENT_ENTITY_EXTRACTION_PROMPT = { + "en": TEMPLATE_ZH, + "zh": TEMPLATE_ZH, + "FORMAT": { + "entity_types": None, # 运行时注入 PMS_ENTITY_TYPES + }, +} diff --git a/graphgen/templates/profiles/pms_policy.v3/atomic.json b/graphgen/templates/profiles/pms_policy.v3/atomic.json new file mode 100644 index 00000000..890a911a --- /dev/null +++ b/graphgen/templates/profiles/pms_policy.v3/atomic.json @@ -0,0 +1,4 @@ +{ + "en": "你是 PMS 项目管理系统的训练题生成器。给定知识图谱片段（节点与关系描述），生成一个可直接执行的“项目查询”问答题。\n\n出题铁律：\n1. 【单项目锚点】问题必须以一个明确项目为对象：题面直接包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。禁止使用“该项目/本项目/该工程”等模糊指代。\n2. 【字段锁定】只允许询问以下信息：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n3. 【禁止反查】不得以人员、部门、组织为起点提问（如“某人属于哪个部门”“某部门有哪些人”“某公司在项目中扮演什么角色”）；人员只能作为“某项目成员名单/某项目某角色由谁担任”的被问内容。\n4. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、管理效果、分工、阶段码含义、组织关系或跨项目比较。\n5. 【事实可溯】答案必须完全来自给定图谱内容；答案给出图谱中的具体值，不得编造。\n6. 【问法自然】使用中文业务口吻提问，问题应像一个真实用户向项目管理系统发起的查询。\n\n7. 【依据引用】support 的 cited 数组必须原样抄录你在“给定图谱片段”中实际依据的行的名称（每行 `- 名称: 描述` 或 `- A - B: 描述` 的名称部分），只列真实依据、不得编造。\n\n输出格式：\nquestion_text\nanswer_text\n{\"cited\": [\"依据的行名称\", ...]}\n\n给定图谱片段：\n{context}\n\n输出：\n", + "zh": "你是 PMS 项目管理系统的训练题生成器。给定知识图谱片段（节点与关系描述），生成一个可直接执行的“项目查询”问答题。\n\n出题铁律：\n1. 【单项目锚点】问题必须以一个明确项目为对象：题面直接包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。禁止使用“该项目/本项目/该工程”等模糊指代。\n2. 【字段锁定】只允许询问以下信息：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n3. 【禁止反查】不得以人员、部门、组织为起点提问（如“某人属于哪个部门”“某部门有哪些人”“某公司在项目中扮演什么角色”）；人员只能作为“某项目成员名单/某项目某角色由谁担任”的被问内容。\n4. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、管理效果、分工、阶段码含义、组织关系或跨项目比较。\n5. 【事实可溯】答案必须完全来自给定图谱内容；答案给出图谱中的具体值，不得编造。\n6. 【问法自然】使用中文业务口吻提问，问题应像一个真实用户向项目管理系统发起的查询。\n\n7. 【依据引用】support 的 cited 数组必须原样抄录你在“给定图谱片段”中实际依据的行的名称（每行 `- 名称: 描述` 或 `- A - B: 描述` 的名称部分），只列真实依据、不得编造。\n\n输出格式：\nquestion_text\nanswer_text\n{\"cited\": [\"依据的行名称\", ...]}\n\n给定图谱片段：\n{context}\n\n输出：\n" +} diff --git a/graphgen/templates/profiles/pms_policy.v3/multi_hop.json b/graphgen/templates/profiles/pms_policy.v3/multi_hop.json new file mode 100644 index 00000000..6f8271ba --- /dev/null +++ b/graphgen/templates/profiles/pms_policy.v3/multi_hop.json @@ -0,0 +1,4 @@ +{ + "en": "你是 PMS 项目管理系统的训练题生成器。给定一个知识子图（实体与关系），生成一个需要两步及以上查询、但仍可由 project-query 技能逐步执行的中文问答对。\n\n出题铁律：\n1. 【项目锚点】问题最终必须落到一个明确项目上；题面包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。\n2. 【可执行多跳】推理链必须锚定在同一个项目上逐步分解为对 project-query 的连续查询：先按完整项目编号定位项目 → 再查该项目的阶段/最终用户/销售工程师/成员角色。不得要求文档检索、外部知识或数值计算。\n3. 【字段锁定】只允许涉及：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n4. 【禁止反查】不得以人员、部门、组织为推理起点反查项目；人员只可作为被问内容（某项目的成员/角色）。\n5. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、分工、组织关系。\n6. 【混合片段守则】给定子图可能混有多个项目的片段。凡是无法确定归属于哪个明确项目的实体或关系，一律不用作出题素材；也不要对比/合并不同项目。\n7. 【事实可溯】答案完全由所锚定项目的子图片段推导，不得编造；答案简洁给出推导结论涉及的具体字段值。\n\n8. 【依据引用】support 的 cited 数组必须原样抄录你在“给定知识子图”中实际依据的行的名称（每行 `- 名称: 描述` 或 `- A - B: 描述` 的名称部分），只列真实依据、不得编造。\n\n输出格式：\nquestion_text\nanswer_text\n{\"cited\": [\"依据的行名称\", ...]}\n\n给定知识子图：\n--实体--\n{entities}\n--关系--\n{relationships}\n\n输出：\n", + "zh": "你是 PMS 项目管理系统的训练题生成器。给定一个知识子图（实体与关系），生成一个需要两步及以上查询、但仍可由 project-query 技能逐步执行的中文问答对。\n\n出题铁律：\n1. 【项目锚点】问题最终必须落到一个明确项目上；题面包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。\n2. 【可执行多跳】推理链必须锚定在同一个项目上逐步分解为对 project-query 的连续查询：先按完整项目编号定位项目 → 再查该项目的阶段/最终用户/销售工程师/成员角色。不得要求文档检索、外部知识或数值计算。\n3. 【字段锁定】只允许涉及：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n4. 【禁止反查】不得以人员、部门、组织为推理起点反查项目；人员只可作为被问内容（某项目的成员/角色）。\n5. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、分工、组织关系。\n6. 【混合片段守则】给定子图可能混有多个项目的片段。凡是无法确定归属于哪个明确项目的实体或关系，一律不用作出题素材；也不要对比/合并不同项目。\n7. 【事实可溯】答案完全由所锚定项目的子图片段推导，不得编造；答案简洁给出推导结论涉及的具体字段值。\n\n8. 【依据引用】support 的 cited 数组必须原样抄录你在“给定知识子图”中实际依据的行的名称（每行 `- 名称: 描述` 或 `- A - B: 描述` 的名称部分），只列真实依据、不得编造。\n\n输出格式：\nquestion_text\nanswer_text\n{\"cited\": [\"依据的行名称\", ...]}\n\n给定知识子图：\n--实体--\n{entities}\n--关系--\n{relationships}\n\n输出：\n" +} From 194f06b6b1a307ff12c6d1402b7a5d2c78adf9c0 Mon Sep 17 00:00:00 2001 From: Leon-L20 Date: Fri, 9 Oct 2026 10:12:11 +0000 Subject: [PATCH 3/9] =?UTF-8?q?fix(pms):=20Phase2=20=E7=9C=9F=E5=AE=9E?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E5=9B=9B=E7=BC=BA=E9=99=B7=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=20+=20=E4=BA=8B=E4=BB=B6/=E5=8F=82=E4=B8=8E=E8=80=85=E7=A1=AE?= =?UTF-8?q?=E5=AE=9A=E6=80=A7=E5=90=88=E5=B9=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - event_contract 抽为纯 stdlib 模块（rft 契约测试可按路径加载） - builder 补回 json 导入；EVENT 节点/成员边/参与者实体确定性 SEP 合并，不再触发 LightRAG LLM 摘要（摘要器曾把 event id 扩写成伪业务事实污染出题上下文） - EventJoinPartitioner 继承 BasePartitioner（community2batch）；models lazy export 补齐 - v3 profile support 大括号转义（str.format KeyError）；分区服务移除 metadata 依赖 （support 校验以持久化分区节点集合为准，JSONL 往返一致） - 真实 10 记录端到端 smoke 全通：7 分区→7 题全锚点合规、cited 全部通过分区校验 Co-Authored-By: Claude Opus 5.5 --- docs/PMS_PATCHES.md | 8 +- .../kg_builder/event_entity_kg_builder.py | 73 ++++++++++++++++++- .../partitioner/event_join_partitioner.py | 5 +- .../operators/generate/generate_service.py | 6 +- .../operators/partition/partition_service.py | 4 - .../profiles/pms_policy.v3/atomic.json | 4 +- .../profiles/pms_policy.v3/multi_hop.json | 4 +- 7 files changed, 89 insertions(+), 15 deletions(-) diff --git a/docs/PMS_PATCHES.md b/docs/PMS_PATCHES.md index 88d2fb8e..f69d7631 100644 --- a/docs/PMS_PATCHES.md +++ b/docs/PMS_PATCHES.md @@ -65,12 +65,16 @@ SAG 思想移植（MIT，arXiv:2606.15971；参考源码 `workspace/sag_referenc - `models/kg_builder/event_contract.py`：纯 stdlib 响应合同（禁合同外字段、is_valid 自报、 无锚 fail-closed 丢弃并计数）+ 数字接地校验（SAG grounding 移植）+ 确定性事件 ID。 - `models/kg_builder/event_entity_kg_builder.py`：事件→星型编码（EVENT 节点 + 成员边， - 边属性含角色）；合并/摘要/SEP 血缘复用官方 LightRAG merge。 + 边属性含角色）。实体描述/普通边继续复用官方合并；**EVENT 节点与事件成员边用确定性 + SEP 合并、不走 LLM 摘要**（SAG 事件是事实文本，不能让 merge summarizer 把 event_id + 扩写成“项目身份/分工总结”污染出题上下文）。事件描述限长 4,000 字符，且原文链以 + `source_id` SEP 保持闭合。 - `operators/build_kg/`：`kg_method: event_entity` 分发（默认 light_rag 与官方逐字节一致）。 - `models/partitioner/event_join_partitioner.py`：共享实体 join + 同锚硬约束的事件簇分区 （SAG join 语义离线物化），产出官方 `Community`，metadata 携带 event_ids。 - `models/generator/support.py` + atomic/multi_hop 解析 + GenerateService 校验： - `{"cited": [...]}` 引用必须存在于本分区节点集合，失败剥离保留题面。 + `{"cited": [...]}` 引用必须存在于本分区节点集合（EVENT 节点名即事件审计 + ID，不依赖自定义 Community metadata 以保证 JSONL cache 往返一致），失败剥离保留题面。 - `templates/profiles/pms_policy.v3/`：v2 模板 + support 输出要求。 ## 验收 diff --git a/graphgen/models/kg_builder/event_entity_kg_builder.py b/graphgen/models/kg_builder/event_entity_kg_builder.py index f247cde0..1aa44489 100644 --- a/graphgen/models/kg_builder/event_entity_kg_builder.py +++ b/graphgen/models/kg_builder/event_entity_kg_builder.py @@ -27,6 +27,9 @@ from graphgen.templates.kg.event_entity_extraction import PMS_ENTITY_TYPES from graphgen.utils import detect_main_language, logger +_EVENT_DESC_CAP = 4000 +_ENTITY_DESC_CAP = 2000 + from graphgen.models.kg_builder.event_contract import ( PROJECT_ID_RE, _ALLOWED_ENTITY_KEYS, @@ -129,7 +132,75 @@ async def extract( return dict(nodes), dict(edges) async def merge_nodes(self, node_data: tuple, kg_instance: BaseGraphStorage) -> dict: - return await self._merger.merge_nodes(node_data, kg_instance=kg_instance) + entity_name, node_list = node_data + if node_list and str(node_list[0].get("entity_type", "")).upper() == "EVENT": + return await self._merge_event_node(entity_name, node_list, kg_instance) + # 星型事件参与者是图投影，不需要再次 LLM 摘要；只做确定性去重合并， + # 避免 LightRAG summarizer 把跨事件上下文扩写成伪业务事实。 + return await self._merge_entity_node(entity_name, node_list, kg_instance) + + async def _merge_entity_node( + self, entity_name: str, node_list: list, kg_instance: BaseGraphStorage + ) -> dict: + existing = kg_instance.get_node(entity_name) or {} + all_rows = list(node_list) + if existing: + all_rows.append(existing) + types = [str(dp.get("entity_type") or "KEYWORD") for dp in all_rows] + entity_type = sorted(set(types), key=lambda t: (-types.count(t), t))[0] + descriptions = sorted({str(dp.get("description") or "") for dp in all_rows if dp.get("description")}) + merged = "".join(descriptions)[:_ENTITY_DESC_CAP] + source_ids = sorted({sid for dp in all_rows for sid in str(dp.get("source_id", "")).split("") if sid}) + payload = { + "entity_type": entity_type, + "entity_name": entity_name, + "description": merged, + "source_id": "".join(source_ids), + "length": self._merger.tokenizer.count_tokens(merged), + } + kg_instance.upsert_node(entity_name, node_data=payload) + return payload + + async def _merge_event_node( + self, entity_name: str, node_list: list, kg_instance: BaseGraphStorage + ) -> dict: + """事件节点确定性合并：SEP 去重拼接（截断），不走 LLM 摘要—— + 官方摘要器会把事件 id 当实体名写进描述，污染出题上下文。""" + merged = "".join(sorted({dp["description"] for dp in node_list})) + if len(merged) > _EVENT_DESC_CAP: + merged = merged[:_EVENT_DESC_CAP] + node_data_dict = { + "entity_type": "EVENT", + "entity_name": entity_name, + "description": merged, + "source_id": "".join( + sorted({sid for dp in node_list for sid in str(dp.get("source_id", "")).split("") if sid} + | {sid for sid in str((kg_instance.get_node(entity_name) or {}).get("source_id", "")).split("") if sid}) + ), + "length": self._merger.tokenizer.count_tokens(merged), + } + kg_instance.upsert_node(entity_name, node_data=node_data_dict) + return node_data_dict async def merge_edges(self, edges_data: tuple, kg_instance: BaseGraphStorage) -> dict: + (src_id, tgt_id), edge_list = edges_data + if src_id.startswith("event:") or tgt_id.startswith("event:"): + # 事件成员边同样确定性合并，不做 LLM 摘要 + merged = "".join(sorted({dp["description"] for dp in edge_list})) + if len(merged) > _EVENT_DESC_CAP: + merged = merged[:_EVENT_DESC_CAP] + source_ids = { + sid for dp in edge_list for sid in str(dp.get("source_id", "")).split("") if sid + } | { + sid for sid in str((kg_instance.get_edge(src_id, tgt_id) or {}).get("source_id", "")).split("") if sid + } + edge_data = { + "src_id": src_id, + "tgt_id": tgt_id, + "description": merged, + "source_id": "".join(sorted(source_ids)), + "length": self._merger.tokenizer.count_tokens(merged), + } + kg_instance.upsert_edge(src_id, tgt_id, edge_data=edge_data) + return edge_data return await self._merger.merge_edges(edges_data, kg_instance=kg_instance) diff --git a/graphgen/models/partitioner/event_join_partitioner.py b/graphgen/models/partitioner/event_join_partitioner.py index 48d51b7b..3414ff16 100644 --- a/graphgen/models/partitioner/event_join_partitioner.py +++ b/graphgen/models/partitioner/event_join_partitioner.py @@ -17,13 +17,13 @@ from collections import deque from typing import Any, Iterable, List, Optional, Set, Tuple -from graphgen.bases import BaseGraphStorage +from graphgen.bases import BaseGraphStorage, BasePartitioner from graphgen.bases.datatypes import Community EVENT_NODE_PREFIX = "event:" -class EventJoinPartitioner: +class EventJoinPartitioner(BasePartitioner): """EVENT 星型子图 → 同项目事件簇（官方 Community 数据类型）。""" def partition( @@ -102,7 +102,6 @@ def _grow(seed: Tuple[str, dict]) -> Optional[Community]: id=seed[0], nodes=member_events + sorted(member_entities), edges=[tuple(sorted(edge)) for edge in sorted(member_edges, key=sorted)], - metadata={"event_ids": member_events, "anchor": anchor}, ) for event_id, data in events: diff --git a/graphgen/operators/generate/generate_service.py b/graphgen/operators/generate/generate_service.py index 2179a1eb..e6676fe6 100644 --- a/graphgen/operators/generate/generate_service.py +++ b/graphgen/operators/generate/generate_service.py @@ -139,7 +139,11 @@ def process(self, batch: list) -> Tuple[list, dict]: continue # PMS fork（Phase 2 升级 C）：.cited 必须全部存在于本分区 # 节点集合，否则整块剥离、题面保留（题面质量与审计质量解耦）。 - node_names = {node[0] for node in item.get("nodes", []) if isinstance(node, (list, tuple)) and node} + node_names = { + node[0] + for node in item.get("nodes", []) + if isinstance(node, (list, tuple)) and node + } for qa_pair in qa_pairs: support = qa_pair.pop("support", None) if isinstance(qa_pair, dict) else None support = validate_support(support, node_names) diff --git a/graphgen/operators/partition/partition_service.py b/graphgen/operators/partition/partition_service.py index 6e7ddff3..db355769 100644 --- a/graphgen/operators/partition/partition_service.py +++ b/graphgen/operators/partition/partition_service.py @@ -91,10 +91,6 @@ def generator(): "nodes": b[0], "edges": b[1], } - # PMS fork（Phase 2）：事件簇附带 event_ids，供出题端 support 校验。 - event_ids = list(getattr(community, "metadata", {}) or {}).get("event_ids") or [] - if event_ids: - result["events"] = event_ids result["_trace_id"] = self.get_trace_id(result) yield result logger.info("Total communities partitioned: %d", count) diff --git a/graphgen/templates/profiles/pms_policy.v3/atomic.json b/graphgen/templates/profiles/pms_policy.v3/atomic.json index 890a911a..1f1bb4ad 100644 --- a/graphgen/templates/profiles/pms_policy.v3/atomic.json +++ b/graphgen/templates/profiles/pms_policy.v3/atomic.json @@ -1,4 +1,4 @@ { - "en": "你是 PMS 项目管理系统的训练题生成器。给定知识图谱片段（节点与关系描述），生成一个可直接执行的“项目查询”问答题。\n\n出题铁律：\n1. 【单项目锚点】问题必须以一个明确项目为对象：题面直接包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。禁止使用“该项目/本项目/该工程”等模糊指代。\n2. 【字段锁定】只允许询问以下信息：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n3. 【禁止反查】不得以人员、部门、组织为起点提问（如“某人属于哪个部门”“某部门有哪些人”“某公司在项目中扮演什么角色”）；人员只能作为“某项目成员名单/某项目某角色由谁担任”的被问内容。\n4. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、管理效果、分工、阶段码含义、组织关系或跨项目比较。\n5. 【事实可溯】答案必须完全来自给定图谱内容；答案给出图谱中的具体值，不得编造。\n6. 【问法自然】使用中文业务口吻提问，问题应像一个真实用户向项目管理系统发起的查询。\n\n7. 【依据引用】support 的 cited 数组必须原样抄录你在“给定图谱片段”中实际依据的行的名称（每行 `- 名称: 描述` 或 `- A - B: 描述` 的名称部分），只列真实依据、不得编造。\n\n输出格式：\nquestion_text\nanswer_text\n{\"cited\": [\"依据的行名称\", ...]}\n\n给定图谱片段：\n{context}\n\n输出：\n", - "zh": "你是 PMS 项目管理系统的训练题生成器。给定知识图谱片段（节点与关系描述），生成一个可直接执行的“项目查询”问答题。\n\n出题铁律：\n1. 【单项目锚点】问题必须以一个明确项目为对象：题面直接包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。禁止使用“该项目/本项目/该工程”等模糊指代。\n2. 【字段锁定】只允许询问以下信息：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n3. 【禁止反查】不得以人员、部门、组织为起点提问（如“某人属于哪个部门”“某部门有哪些人”“某公司在项目中扮演什么角色”）；人员只能作为“某项目成员名单/某项目某角色由谁担任”的被问内容。\n4. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、管理效果、分工、阶段码含义、组织关系或跨项目比较。\n5. 【事实可溯】答案必须完全来自给定图谱内容；答案给出图谱中的具体值，不得编造。\n6. 【问法自然】使用中文业务口吻提问，问题应像一个真实用户向项目管理系统发起的查询。\n\n7. 【依据引用】support 的 cited 数组必须原样抄录你在“给定图谱片段”中实际依据的行的名称（每行 `- 名称: 描述` 或 `- A - B: 描述` 的名称部分），只列真实依据、不得编造。\n\n输出格式：\nquestion_text\nanswer_text\n{\"cited\": [\"依据的行名称\", ...]}\n\n给定图谱片段：\n{context}\n\n输出：\n" + "en": "你是 PMS 项目管理系统的训练题生成器。给定知识图谱片段（节点与关系描述），生成一个可直接执行的“项目查询”问答题。\n\n出题铁律：\n1. 【单项目锚点】问题必须以一个明确项目为对象：题面直接包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。禁止使用“该项目/本项目/该工程”等模糊指代。\n2. 【字段锁定】只允许询问以下信息：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n3. 【禁止反查】不得以人员、部门、组织为起点提问（如“某人属于哪个部门”“某部门有哪些人”“某公司在项目中扮演什么角色”）；人员只能作为“某项目成员名单/某项目某角色由谁担任”的被问内容。\n4. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、管理效果、分工、阶段码含义、组织关系或跨项目比较。\n5. 【事实可溯】答案必须完全来自给定图谱内容；答案给出图谱中的具体值，不得编造。\n6. 【问法自然】使用中文业务口吻提问，问题应像一个真实用户向项目管理系统发起的查询。\n\n7. 【依据引用】答案不得把事件节点 ID 当作项目实体或名称。support 的 cited 数组必须原样抄录你在“给定图谱片段”中实际依据的行的名称（每行 `- 名称: 描述` 或 `- A - B: 描述` 的名称部分），只列真实依据、不得编造。\n\n输出格式：\nquestion_text\nanswer_text\n{{\"cited\": [\"依据的行名称\", ...]}}\n\n给定图谱片段：\n{context}\n\n输出：\n", + "zh": "你是 PMS 项目管理系统的训练题生成器。给定知识图谱片段（节点与关系描述），生成一个可直接执行的“项目查询”问答题。\n\n出题铁律：\n1. 【单项目锚点】问题必须以一个明确项目为对象：题面直接包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。禁止使用“该项目/本项目/该工程”等模糊指代。\n2. 【字段锁定】只允许询问以下信息：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n3. 【禁止反查】不得以人员、部门、组织为起点提问（如“某人属于哪个部门”“某部门有哪些人”“某公司在项目中扮演什么角色”）；人员只能作为“某项目成员名单/某项目某角色由谁担任”的被问内容。\n4. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、管理效果、分工、阶段码含义、组织关系或跨项目比较。\n5. 【事实可溯】答案必须完全来自给定图谱内容；答案给出图谱中的具体值，不得编造。\n6. 【问法自然】使用中文业务口吻提问，问题应像一个真实用户向项目管理系统发起的查询。\n\n7. 【依据引用】答案不得把事件节点 ID 当作项目实体或名称。support 的 cited 数组必须原样抄录你在“给定图谱片段”中实际依据的行的名称（每行 `- 名称: 描述` 或 `- A - B: 描述` 的名称部分），只列真实依据、不得编造。\n\n输出格式：\nquestion_text\nanswer_text\n{{\"cited\": [\"依据的行名称\", ...]}}\n\n给定图谱片段：\n{context}\n\n输出：\n" } diff --git a/graphgen/templates/profiles/pms_policy.v3/multi_hop.json b/graphgen/templates/profiles/pms_policy.v3/multi_hop.json index 6f8271ba..6bd06a1a 100644 --- a/graphgen/templates/profiles/pms_policy.v3/multi_hop.json +++ b/graphgen/templates/profiles/pms_policy.v3/multi_hop.json @@ -1,4 +1,4 @@ { - "en": "你是 PMS 项目管理系统的训练题生成器。给定一个知识子图（实体与关系），生成一个需要两步及以上查询、但仍可由 project-query 技能逐步执行的中文问答对。\n\n出题铁律：\n1. 【项目锚点】问题最终必须落到一个明确项目上；题面包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。\n2. 【可执行多跳】推理链必须锚定在同一个项目上逐步分解为对 project-query 的连续查询：先按完整项目编号定位项目 → 再查该项目的阶段/最终用户/销售工程师/成员角色。不得要求文档检索、外部知识或数值计算。\n3. 【字段锁定】只允许涉及：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n4. 【禁止反查】不得以人员、部门、组织为推理起点反查项目；人员只可作为被问内容（某项目的成员/角色）。\n5. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、分工、组织关系。\n6. 【混合片段守则】给定子图可能混有多个项目的片段。凡是无法确定归属于哪个明确项目的实体或关系，一律不用作出题素材；也不要对比/合并不同项目。\n7. 【事实可溯】答案完全由所锚定项目的子图片段推导，不得编造；答案简洁给出推导结论涉及的具体字段值。\n\n8. 【依据引用】support 的 cited 数组必须原样抄录你在“给定知识子图”中实际依据的行的名称（每行 `- 名称: 描述` 或 `- A - B: 描述` 的名称部分），只列真实依据、不得编造。\n\n输出格式：\nquestion_text\nanswer_text\n{\"cited\": [\"依据的行名称\", ...]}\n\n给定知识子图：\n--实体--\n{entities}\n--关系--\n{relationships}\n\n输出：\n", - "zh": "你是 PMS 项目管理系统的训练题生成器。给定一个知识子图（实体与关系），生成一个需要两步及以上查询、但仍可由 project-query 技能逐步执行的中文问答对。\n\n出题铁律：\n1. 【项目锚点】问题最终必须落到一个明确项目上；题面包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。\n2. 【可执行多跳】推理链必须锚定在同一个项目上逐步分解为对 project-query 的连续查询：先按完整项目编号定位项目 → 再查该项目的阶段/最终用户/销售工程师/成员角色。不得要求文档检索、外部知识或数值计算。\n3. 【字段锁定】只允许涉及：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n4. 【禁止反查】不得以人员、部门、组织为推理起点反查项目；人员只可作为被问内容（某项目的成员/角色）。\n5. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、分工、组织关系。\n6. 【混合片段守则】给定子图可能混有多个项目的片段。凡是无法确定归属于哪个明确项目的实体或关系，一律不用作出题素材；也不要对比/合并不同项目。\n7. 【事实可溯】答案完全由所锚定项目的子图片段推导，不得编造；答案简洁给出推导结论涉及的具体字段值。\n\n8. 【依据引用】support 的 cited 数组必须原样抄录你在“给定知识子图”中实际依据的行的名称（每行 `- 名称: 描述` 或 `- A - B: 描述` 的名称部分），只列真实依据、不得编造。\n\n输出格式：\nquestion_text\nanswer_text\n{\"cited\": [\"依据的行名称\", ...]}\n\n给定知识子图：\n--实体--\n{entities}\n--关系--\n{relationships}\n\n输出：\n" + "en": "你是 PMS 项目管理系统的训练题生成器。给定一个知识子图（实体与关系），生成一个需要两步及以上查询、但仍可由 project-query 技能逐步执行的中文问答对。\n\n出题铁律：\n1. 【项目锚点】问题最终必须落到一个明确项目上；题面包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。\n2. 【可执行多跳】推理链必须锚定在同一个项目上逐步分解为对 project-query 的连续查询：先按完整项目编号定位项目 → 再查该项目的阶段/最终用户/销售工程师/成员角色。不得要求文档检索、外部知识或数值计算。\n3. 【字段锁定】只允许涉及：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n4. 【禁止反查】不得以人员、部门、组织为推理起点反查项目；人员只可作为被问内容（某项目的成员/角色）。\n5. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、分工、组织关系。\n6. 【混合片段守则】给定子图可能混有多个项目的片段。凡是无法确定归属于哪个明确项目的实体或关系，一律不用作出题素材；也不要对比/合并不同项目。\n7. 【事实可溯】答案完全由所锚定项目的子图片段推导，不得编造；答案简洁给出推导结论涉及的具体字段值。\n\n8. 【依据引用】答案不得把事件节点 ID 当作项目实体或名称。support 的 cited 数组必须原样抄录你在“给定知识子图”中实际依据的行的名称（每行 `- 名称: 描述` 或 `- A - B: 描述` 的名称部分），只列真实依据、不得编造。\n\n输出格式：\nquestion_text\nanswer_text\n{{\"cited\": [\"依据的行名称\", ...]}}\n\n给定知识子图：\n--实体--\n{entities}\n--关系--\n{relationships}\n\n输出：\n", + "zh": "你是 PMS 项目管理系统的训练题生成器。给定一个知识子图（实体与关系），生成一个需要两步及以上查询、但仍可由 project-query 技能逐步执行的中文问答对。\n\n出题铁律：\n1. 【项目锚点】问题最终必须落到一个明确项目上；题面包含该项目的完整项目编号（如 P-24084947、I-26015101）。项目名称在同系统中可能不唯一，禁止只凭名称指代项目；若确需在题面提到名称，必须连同完整项目编号一起出现。\n2. 【可执行多跳】推理链必须锚定在同一个项目上逐步分解为对 project-query 的连续查询：先按完整项目编号定位项目 → 再查该项目的阶段/最终用户/销售工程师/成员角色。不得要求文档检索、外部知识或数值计算。\n3. 【字段锁定】只允许涉及：项目编号、项目名称、项目阶段、最终用户、销售工程师、项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件一律禁止。\n4. 【禁止反查】不得以人员、部门、组织为推理起点反查项目；人员只可作为被问内容（某项目的成员/角色）。\n5. 【禁止越界语义】不得询问职责解释、工作内容、产能、交付物、分工、组织关系。\n6. 【混合片段守则】给定子图可能混有多个项目的片段。凡是无法确定归属于哪个明确项目的实体或关系，一律不用作出题素材；也不要对比/合并不同项目。\n7. 【事实可溯】答案完全由所锚定项目的子图片段推导，不得编造；答案简洁给出推导结论涉及的具体字段值。\n\n8. 【依据引用】答案不得把事件节点 ID 当作项目实体或名称。support 的 cited 数组必须原样抄录你在“给定知识子图”中实际依据的行的名称（每行 `- 名称: 描述` 或 `- A - B: 描述` 的名称部分），只列真实依据、不得编造。\n\n输出格式：\nquestion_text\nanswer_text\n{{\"cited\": [\"依据的行名称\", ...]}}\n\n给定知识子图：\n--实体--\n{entities}\n--关系--\n{relationships}\n\n输出：\n" } From c5ef453aa661776b757902a990c529fdab2bfc9a Mon Sep 17 00:00:00 2001 From: Leon-L20 Date: Sat, 10 Oct 2026 10:11:04 +0000 Subject: [PATCH 4/9] =?UTF-8?q?feat:=20=E9=80=9A=E7=94=A8=E5=8C=96GraphGen?= =?UTF-8?q?=E4=BA=8B=E4=BB=B6=E6=8A=BD=E5=8F=96=E5=B9=B6=E5=AE=8C=E5=96=84?= =?UTF-8?q?=E5=A4=9A=E9=A1=B9=E7=9B=AE=E5=8F=91=E5=B8=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 引入版本化事件抽取profile，将实体类型、prompt、锚点和数字接地策略从PMS专用代码解耦。event_join基于显式锚点属性隔离，生成prompt隐藏内部事件ID并把support引用映射回canonical节点。\n\n新增多项目SDK与不可变tag发布文档，修正安装包资源声明和过时的partition integration测试；保留LightRAG+ECE默认路径及PMS事件ID兼容。完成真实非PMS抽取、分区和问答生成smoke，默认生成结果未泄漏内部事件ID。\n\nCo-Authored-By: Claude Opus 5.5 --- MANIFEST.in | 2 + README.md | 6 + README_zh.md | 6 + docs/MULTI_PROJECT_SDK.md | 84 ++++++++++++ docs/PMS_PATCHES.md | 15 +- docs/RELEASE_NOTES-1.0.0.md | 23 ++++ docs/RELEASING.md | 43 ++++++ graphgen/_version.py | 2 +- graphgen/models/generator/support.py | 87 ++++++++++-- graphgen/models/kg_builder/event_contract.py | 93 ++++++++----- .../kg_builder/event_entity_kg_builder.py | 105 +++++++------- .../partitioner/event_join_partitioner.py | 38 +++--- graphgen/operators/build_kg/build_event_kg.py | 17 ++- .../operators/build_kg/build_kg_service.py | 9 +- .../operators/generate/generate_service.py | 20 +-- .../operators/partition/partition_service.py | 5 + graphgen/templates/event_profile_loader.py | 129 ++++++++++++++++++ .../example_event.v1/profile.json | 13 ++ .../example_event.v1/prompt.zh.txt | 28 ++++ .../generic_event.v1/profile.json | 12 ++ .../generic_event.v1/prompt.zh.txt | 27 ++++ .../event_profiles/pms_event.v1/profile.json | 18 +++ .../event_profiles/pms_event.v1/prompt.zh.txt | 74 ++++++++++ .../templates/kg/event_entity_extraction.py | 118 ++-------------- requirements-dev.txt | 2 +- setup.py | 14 +- .../partitioner/test_bfs_partitioner.py | 32 ++--- .../partitioner/test_dfs_partitioner.py | 34 ++--- .../partitioner/test_ece_partitioner.py | 65 ++++----- tests/test_event_profiles.py | 20 +++ tests/test_package_profiles.py | 30 ++++ 31 files changed, 852 insertions(+), 319 deletions(-) create mode 100644 docs/MULTI_PROJECT_SDK.md create mode 100644 docs/RELEASE_NOTES-1.0.0.md create mode 100644 docs/RELEASING.md create mode 100644 graphgen/templates/event_profile_loader.py create mode 100644 graphgen/templates/event_profiles/example_event.v1/profile.json create mode 100644 graphgen/templates/event_profiles/example_event.v1/prompt.zh.txt create mode 100644 graphgen/templates/event_profiles/generic_event.v1/profile.json create mode 100644 graphgen/templates/event_profiles/generic_event.v1/prompt.zh.txt create mode 100644 graphgen/templates/event_profiles/pms_event.v1/profile.json create mode 100644 graphgen/templates/event_profiles/pms_event.v1/prompt.zh.txt create mode 100644 tests/test_event_profiles.py create mode 100644 tests/test_package_profiles.py diff --git a/MANIFEST.in b/MANIFEST.in index d1a9003b..c8572113 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,5 @@ include requirements.txt include graphgen/configs/*.yaml recursive-include graphgen/configs *.yaml +recursive-include graphgen/templates/profiles *.json +recursive-include graphgen/templates/event_profiles *.json *.txt diff --git a/README.md b/README.md index 2fdff54b..86e5d4fe 100644 --- a/README.md +++ b/README.md @@ -374,3 +374,9 @@ This project is licensed under the [Apache License 2.0](LICENSE). ## 📅 Star History [![Star History Chart](https://star-history.dera.page/svg?repos=InternScience/GraphGen&type=Date)](https://star-history.dera.page/#InternScience/GraphGen&Date) + +## Maintained fork: multi-project SDK and event profiles + +This repository also maintains the `Leon-Algo/GraphGen` fork. Fork-specific reusable capabilities, project-owned event extraction profiles, installation guidance, and compatibility boundaries are documented in [docs/MULTI_PROJECT_SDK.md](docs/MULTI_PROJECT_SDK.md). The immutable version/tag and release verification policy is in [docs/RELEASING.md](docs/RELEASING.md); the fork patch ledger and SAG attribution are in [docs/PMS_PATCHES.md](docs/PMS_PATCHES.md). + +Fork profiles are opt-in. The default LightRAG + ECE workflow remains unchanged, and PMS-specific prompts are not a default policy for other projects. diff --git a/README_zh.md b/README_zh.md index fc1bc8e0..6938b754 100644 --- a/README_zh.md +++ b/README_zh.md @@ -375,3 +375,9 @@ GraphGen 首先根据源文本构建细粒度的知识图谱，然后利用期 [![Star History Chart](https://star-history.dera.page/svg?repos=InternScience/GraphGen&type=Date)](https://star-history.dera.page/#InternScience/GraphGen&Date) + +## 自维护 Fork：多项目 SDK 与事件 Profile + +本仓库同时维护 `Leon-Algo/GraphGen` fork。跨项目可复用能力、项目自有事件抽取 profile、安装与兼容边界见[多项目 SDK 指南](docs/MULTI_PROJECT_SDK.md)；不可变版本/tag 与发布验证规则见[发布规范](docs/RELEASING.md)；fork 补丁及 SAG 来源归属见[补丁台账](docs/PMS_PATCHES.md)。 + +Fork 能力均显式 opt-in：默认 LightRAG + ECE 管线保持不变，PMS 专用提示词不会自动成为其他项目的策略。 diff --git a/docs/MULTI_PROJECT_SDK.md b/docs/MULTI_PROJECT_SDK.md new file mode 100644 index 00000000..ee265255 --- /dev/null +++ b/docs/MULTI_PROJECT_SDK.md @@ -0,0 +1,84 @@ +# GraphGen Fork: reusable Graph + SAG event capabilities + +This repository is the independently maintained `Leon-Algo/GraphGen` fork of GraphGen. It remains compatible with the upstream project where documented, while shipping opt-in fork capabilities. The upstream PR is not a release dependency: projects consume reviewed fork commits/tags directly. + +## What is reusable + +- **GraphGen foundation:** pipeline/operators, storage integration, generators, and the default LightRAG + ECE route. +- **Fork capabilities:** versioned generation prompt profiles, output gates, generation concurrency controls, and the opt-in event/entity star-graph builder plus event-join partitioner. +- **SAG-derived event mechanics:** event extraction response contract, numeric grounding, star encoding (one event node linked to participants), deterministic merging, and shared-participant joins. Adapted assets and code retain their upstream license/source attribution in `docs/PMS_PATCHES.md`. + +These mechanisms do not define a universal business ontology. A consuming project selects a versioned event extraction profile and its own generation prompt profile. The `pms_event.v1` and `pms_policy.v2/v3` profiles are PMS-specific examples and are never implicitly applied to unrelated projects. `generic_event.v1` and `example_event.v1` are synthetic examples, not validated domain policies. + +## Install a reviewed fork version + +For a reproducible project, pin an immutable commit in the consuming project's `pyproject.toml` and commit its lock file: + +```toml +[project] +dependencies = [ + "graphg @ git+https://github.com/Leon-Algo/GraphGen.git@", +] +``` + +Resolve and lock with `uv lock` (or add it with `uv add`). Avoid moving branch refs in production. For short-lived development only, install `git+https://github.com/Leon-Algo/GraphGen.git@pms/main`. The distribution name is `graphg`; Python imports use `graphgen`. + +## Select event extraction explicitly + +The default `kg_method` remains `light_rag`; the normal ECE partitioner remains unchanged. To use the event route, configure the build node explicitly: + +```yaml +- id: build_kg + op_name: build_kg + type: map + params: + kg_method: event_entity + event_profile: generic_event.v1 + +- id: partition + op_name: partition + type: map + params: + method: event_join + method_params: + event_entity_type: EVENT + anchor_attribute: anchor + require_same_anchor: false + max_events_per_community: 6 + max_tokens_per_community: 4096 +``` + +For a project-specific profile, copy a profile directory into a project-controlled location and point `event_profile` to the absolute or explicitly configured `profile.json` path; do not rely on the process working directory. Never edit a profile already used by a released run. The loader validates the manifest and prompt, fails closed on missing/invalid inputs, and records SHA-256 fingerprints on the loaded profile object. + +An event profile contains: + +- `schema_version` and immutable `profile_id`; +- `prompt_file` with the required `{input_text}` placeholder; +- `entity_types` allowlist used both in prompt input and response validation; +- `anchor` with required/optional status, optional regex, and optional anchor entity type; +- `grounding.number_grounding`; +- `graph.event_entity_type` and `graph.legacy_event_id`. + +Regex capture group 1 is the anchor when present; otherwise the complete match is used. Event nodes carry the extracted anchor as an explicit string attribute; the partitioner never parses it from the node ID. `require_same_anchor: true` rejects missing anchors and prevents joins across anchor values. Set it to false only when cross-event joining without anchor isolation is acceptable. Limits remain explicit partition parameters. + +The packaged `generic_event.v1` profile demonstrates an anchor-free schema; `example_event.v1` demonstrates a custom anchor and domain entity set. They are fixtures, not a claim that a particular project's extraction policy has been validated. PMS workflows should explicitly select `pms_event.v1` and the corresponding PMS generation policy; legacy configurations with `kg_method: event_entity` and no profile retain the PMS-compatible profile. + +## Generation profiles and gates + +Generation profiles use the independent `prompt_profile` parameter (built-in profile ID or explicit profile directory); do not assume it selects event extraction policy. PMS output gate `pms_project_anchor` is opt-in and is not a generic default. Each project should own its prompt policy and only enable domain gates it has validated. + +## Profile authoring checklist + +1. Define a versioned profile ID and the consuming project's allowed entity types. +2. Write prompt instructions for that domain; include `{input_text}` exactly once or as required by the application. +3. Set an explicit anchor regex or `required: false`; set `anchor_entity_type` only if it is in the entity allowlist. +4. Decide numeric grounding and whether event_join must enforce same-anchor isolation. +5. Test supported, missing, malformed, ungrounded, and unknown-entity responses with a fake LLM before real model smoke. +6. Test extraction → star graph → partition on representative project data, then pin the profile bytes and GraphGen commit in the consuming project. + +## Upgrade and compatibility policy + +- A released tag and profile directory are immutable. Fixes and policy changes get a new commit and a new tag/profile ID; do not retarget tags or edit released profile contents in place. +- Consumers pin a full commit SHA or immutable release tag and commit their lock file. Upgrade by changing the pin deliberately, rerunning tests/smokes, and reviewing GraphGen's release notes and `docs/PMS_PATCHES.md`. +- `light_rag` remains the default. `event_entity` is opt-in. PMS profile defaults exist only for backward compatibility of the PMS fork integration; new projects should always specify their profile explicitly. +- Fork release/tag format, validation gate, attribution, upstream base, and exact release evidence are maintained in `docs/RELEASING.md`. diff --git a/docs/PMS_PATCHES.md b/docs/PMS_PATCHES.md index f69d7631..54736cad 100644 --- a/docs/PMS_PATCHES.md +++ b/docs/PMS_PATCHES.md @@ -60,8 +60,7 @@ SAG 思想移植（MIT，arXiv:2606.15971；参考源码 `workspace/sag_reference/`，方案见 PMS 主仓 `docs/2026-09-18-graphgen-phase2-sag-event-entity-plan.md` §3.2 复用地图）： -- `templates/kg/event_entity_extraction.py`：事件抽取提示词（SAG extract_document v3.1 改编 + - PMS 锚点/字段锁定/数字接地铁律）。 +- `templates/kg/event_entity_extraction.py` 保留旧 Python 常量的兼容导出；实际策略由版本化 event profile 资源提供。 - `models/kg_builder/event_contract.py`：纯 stdlib 响应合同（禁合同外字段、is_valid 自报、 无锚 fail-closed 丢弃并计数）+ 数字接地校验（SAG grounding 移植）+ 确定性事件 ID。 - `models/kg_builder/event_entity_kg_builder.py`：事件→星型编码（EVENT 节点 + 成员边， @@ -77,10 +76,20 @@ SAG 思想移植（MIT，arXiv:2606.15971；参考源码 `workspace/sag_referenc ID，不依赖自定义 Community metadata 以保证 JSONL cache 往返一致），失败剥离保留题面。 - `templates/profiles/pms_policy.v3/`：v2 模板 + support 输出要求。 +## #8 通用事件抽取 Profile 与多项目 SAG 能力 + +- 将事件抽取策略从 builder/契约代码中抽离为 `templates/event_profiles//` 下版本化 manifest + prompt。内置 `pms_event.v1` 保留原始 PMS prompt、8 类实体、P-/I- 锚点和旧事件 ID；`generic_event.v1` 与 `example_event.v1` 是机制示例/测试 fixture，不代表外部领域已验证策略。 +- `templates/event_profile_loader.py` 校验 schema、实体类型、锚点 regex、prompt 占位符及图设置，暴露 manifest/prompt SHA-256；显式无效 profile fail-closed。profile 从包资源读取，不依赖当前工作目录。 +- `build_kg.params.event_profile` 传入事件 builder。老 PMS 配置缺省解析为 `pms_event.v1`；新项目应显式传入自己的 profile。`light_rag` 默认保持不变。 +- `event_contract.py` 仍保证严格响应 JSON、实体 allowlist 和可选数字接地；实体类型、锚点是否必需及锚点模式由 profile 驱动。无锚模式不自动合成 PROJECT/其他锚点实体。 +- 事件节点写入显式 `anchor` 属性。`event_join` 按 `event_entity_type` 和 `anchor_attribute` 配置工作，`require_same_anchor` 控制同锚硬隔离；不再从 ID 推断锚点。PMS profile 使用旧 ID 以兼容既有图缓存，新 profile 使用不编码 domain anchor 的 ID。 +- `setup.py` 与 `MANIFEST.in` 将生成 profile JSON、事件 manifest/prompt 作为 `graphgen` wheel/sdist 包资源。 +- 多项目安装、profile authoring 与升级见 `docs/MULTI_PROJECT_SDK.md`；fork 不可变 tag 与发布门禁见 `docs/RELEASING.md`。 + ## 验收 - 默认路径（无 profile/gate/并发参数）与官方行为逐字节一致； - PMS 主仓库 `tests/test_pms_graphgen_policy_generation.py`、 `tests/test_run_graphgen_defaults.py` 为契约回归（rft 环境按路径加载 纯 stdlib 模块校验）； -- 真实生成由 GraphGen venv smoke 验收。 +- 事件 profile 合同和真实生成由 GraphGen venv smoke 验收。 diff --git a/docs/RELEASE_NOTES-1.0.0.md b/docs/RELEASE_NOTES-1.0.0.md new file mode 100644 index 00000000..4297347c --- /dev/null +++ b/docs/RELEASE_NOTES-1.0.0.md @@ -0,0 +1,23 @@ +# GraphGen Fork 1.0.0 — initial multi-project SDK release + +Tag: `graphgen-fork-v1.0.0` (to be created only after passing release validation). + +Distribution version: `1.0.0` (`graphg`). Upstream base: `InternScience/GraphGen` commit `3a3eb097318d34b07f6f31cb60de5b20712d65ce`. + +## Highlights + +- Profile-driven event/entity extraction: projects supply entity types, prompt, anchor policy, numeric grounding and event graph identity without patching GraphGen source. +- Built-in `pms_event.v1` preserves PMS extraction behavior and legacy event IDs; `generic_event.v1` and `example_event.v1` are synthetic examples only. +- `event_join` reads explicit anchor node attributes and supports opt-in same-anchor isolation independently of event ID encoding. +- Default LightRAG + ECE remains unchanged; event extraction is opt-in. +- Event and generation prompt profile assets are included in wheel and sdist and load through package resources. +- See [multi-project SDK guide](MULTI_PROJECT_SDK.md), [release policy](RELEASING.md), and [patch ledger](PMS_PATCHES.md). + +## Compatibility and validation record + +GraphGen partition integration: 12 passed after correcting tests to the synchronous storage/partitioner APIs. Profile, fake-LLM builder/partition, and package tests: 12 passed. PMS contract regression: `tests/test_pms_graphgen_policy_generation.py` — 23 passed, 1 skipped. +A real one-chunk GraphGen run using the configured PMS launcher/model pool completed extraction → event_join → COT generation. The synthetic non-PMS event anchored on `CASE-A7`; the graph contained the event and two participants; the final QA identified the operator and completed mission, included no internal event hash, and the process exited 0. The run used a synthetic fixture and validates integration mechanics, not any production project's domain accuracy. + +The existing GraphGen partitioner integration suite could not collect because its tests import an unexported `NetworkXStorage` from `graphgen.models`. An unrelated GraphGen extraction e2e test failed at Ray GCS startup (`Could not read 'temp_dir' from GCS`). These environment/baseline failures are not represented as passing tests. + +SAG-derived code/prompt attribution and MIT licensing are tracked in `docs/PMS_PATCHES.md`. diff --git a/docs/RELEASING.md b/docs/RELEASING.md new file mode 100644 index 00000000..44c02f7f --- /dev/null +++ b/docs/RELEASING.md @@ -0,0 +1,43 @@ +# GraphGen fork release and tagging policy + +The fork is an independently versioned Python distribution (`graphg`) maintained at `Leon-Algo/GraphGen`. Upstream merge status does not control fork releases. Release tags are immutable identifiers for exact commits; never move or reuse a published tag. + +## Tag format + +Use `graphgen-fork-vMAJOR.MINOR.PATCH`: + +- **MAJOR**: breaking config/profile/schema or graph identity changes requiring consumer migration. +- **MINOR**: backward-compatible capability or new profile/API. +- **PATCH**: backward-compatible bug fix or documentation/packaging correction. + +PMS domain profiles are independently versioned by their profile IDs and are not implied by a package release. Profiles referenced by production configs must not be edited after release; changes get a new profile ID. + +## Release gate + +Before tagging: + +1. Review the fork diff and working tree; ensure no unrelated user changes are included. +2. Run deterministic profile/contract/builder/partitioner tests, relevant upstream GraphGen tests, and PMS compatibility tests. +3. Run a non-PMS extraction → star graph → event_join smoke and a PMS compatibility smoke; confirm default LightRAG + ECE behavior is unchanged. +4. Build wheel and sdist in a disposable directory; inspect both for prompt/profile assets and install the wheel into a clean environment to load each built-in profile. +5. Run `git diff --check`, record exact commit, test commands/results, profile hashes and upstream base. + +If any required gate fails, do not tag or push the release. + +## Release sequence + +1. Merge only reviewed release files into the intended fork branch; preserve unrelated local changes. +2. Commit with a concise message and verify the final commit SHA and clean release diff. +3. Confirm the desired tag is unused locally and on the fork remote. +4. Create the tag on that exact commit, then push the intended branch and tag to the `fork` remote (`https://github.com/Leon-Algo/GraphGen.git`). Never push the fork release tag to upstream by accident. +5. Publish release notes that include: distribution version, tag and commit SHA, upstream base commit, key behavior/config changes, profile IDs and asset hashes, compatibility/migration notes, license/source attribution, test and smoke evidence, and any known limitations. + +Example commands (replace placeholders only after verification): + +```bash +git tag -a graphgen-fork-v1.0.0 -m "GraphGen fork 1.0.0" +git push fork pms/main +git push fork graphgen-fork-v1.0.0 +``` + +Consumers should pin the release tag or preferably full commit SHA and lock their environment. Tag creation/push is an externally visible release action and must occur only after the release gate passes and the maintainer authorizes publication. diff --git a/graphgen/_version.py b/graphgen/_version.py index a7ae27a2..43fe2c19 100644 --- a/graphgen/_version.py +++ b/graphgen/_version.py @@ -1,6 +1,6 @@ from typing import Tuple -__version__ = "0.1.0.post20250930" +__version__ = "1.0.0" short_version = __version__ diff --git a/graphgen/models/generator/support.py b/graphgen/models/generator/support.py index 810f65d2..d66dd633 100644 --- a/graphgen/models/generator/support.py +++ b/graphgen/models/generator/support.py @@ -1,10 +1,4 @@ -""" 审计块解析（PMS fork，Phase 2 升级 C）。 - -生成器可在 / 之后输出 -``{"cited": ["节点或事件名称", ...]}``； -本模块只负责解析，有效性校验（cited 是否真存在于出题分区）在 -GenerateService.process 中按分区的节点集合执行，失败剥离 support 保留题面。 -""" +"""Parse support blocks and build a display-safe graph view for generators.""" from __future__ import annotations @@ -16,7 +10,7 @@ def parse_support(response: str) -> Optional[dict[str, Any]]: - """从模型响应解析 support 块；缺失/畸形返回 None（不影响题面解析）。""" + """Parse support XML; malformed or absent blocks are ignored.""" match = _SUPPORT_RE.search(response or "") if not match: return None @@ -24,16 +18,85 @@ def parse_support(response: str) -> Optional[dict[str, Any]]: payload = json.loads(match.group(1).strip()) except json.JSONDecodeError: return None - if not isinstance(payload, dict) or set(payload.keys()) != {"cited"}: + if not isinstance(payload, dict) or set(payload) != {"cited"}: return None cited = payload.get("cited") - if not isinstance(cited, list) or not all(isinstance(c, str) and c.strip() for c in cited): + if not isinstance(cited, list) or not all(isinstance(value, str) and value.strip() for value in cited): return None - return {"cited": [c.strip() for c in cited]} + return {"cited": [value.strip() for value in cited]} + + +def _is_event_node(node: tuple[str, dict]) -> bool: + return str(node[1].get("graph_role", "")).lower() == "event" + + +def _event_display_name(node: tuple[str, dict]) -> str: + title = str(node[1].get("description", "")).split("：", 1)[0].strip() + return title or "事件" + + +def build_generation_view(item: dict[str, Any]) -> dict[str, Any]: + """Replace internal event IDs with readable names for generation only. + + Canonical node/edge records remain unchanged for lineage. Duplicate event + titles receive deterministic suffixes; support names are mapped back and + validated against the original partition node IDs. + """ + original_nodes = [node for node in item.get("nodes", []) if isinstance(node, (list, tuple)) and len(node) == 2] + event_nodes = [node for node in original_nodes if _is_event_node(node)] + used_names = { + str(node[0]) for node in original_nodes if not _is_event_node(node) + } + display_by_id: dict[str, str] = {} + for node in sorted(event_nodes, key=lambda row: str(row[0])): + base = _event_display_name(node) + display = base + suffix = 2 + while display in used_names: + display = f"{base} ({suffix})" + suffix += 1 + used_names.add(display) + display_by_id[str(node[0])] = display + + nodes = [ + (display_by_id.get(str(node_id), node_id), data) + for node_id, data in original_nodes + ] + edges = [] + support_aliases = {name: node_id for node_id, name in display_by_id.items()} + for edge in item.get("edges", []): + if not isinstance(edge, (list, tuple)) or len(edge) != 3: + continue + source, target, data = edge + display_source = display_by_id.get(str(source), source) + display_target = display_by_id.get(str(target), target) + edges.append((display_source, display_target, data)) + if str(source) in display_by_id: + support_aliases[f"{display_source} - {display_target}"] = f"{source} - {target}" + elif str(target) in display_by_id: + support_aliases[f"{display_source} - {display_target}"] = f"{source} - {target}" + + def resolve_support(support: Optional[dict[str, Any]]) -> Optional[dict[str, Any]]: + if not support: + return None + resolved = [] + for cited in support.get("cited", []): + if cited in support_aliases: + resolved.append(support_aliases[cited]) + else: + resolved.append(cited) + return {"cited": resolved} + + return { + "nodes": nodes, + "edges": edges, + "canonical_nodes": [str(node_id) for node_id, _ in original_nodes], + "resolve_support": resolve_support, + } def validate_support(support: Optional[dict[str, Any]], node_names: set[str]) -> Optional[dict[str, Any]]: - """cited 全部存在于分区节点集合 → 原样返回；否则返回 None（剥离）。""" + """Keep support only when all citations point to existing partition nodes.""" if not support: return None cited = support.get("cited") or [] diff --git a/graphgen/models/kg_builder/event_contract.py b/graphgen/models/kg_builder/event_contract.py index 374ea557..d6cf7350 100644 --- a/graphgen/models/kg_builder/event_contract.py +++ b/graphgen/models/kg_builder/event_contract.py @@ -1,10 +1,11 @@ -"""Pure stdlib contracts for SAG-adapted event extraction (testable without GraphGen).""" +"""Pure stdlib contracts for profile-driven SAG event extraction.""" from __future__ import annotations + import hashlib import json import re from collections import defaultdict -from typing import Any, Optional +from typing import Any PMS_ENTITY_TYPES: list[dict] = [ {"type": "PROJECT", "description": "项目本体，含完整项目编号"}, @@ -16,36 +17,53 @@ {"type": "DATE", "description": "日期或时间"}, {"type": "METRIC", "description": "数量、比例等指标"}, ] - PROJECT_ID_RE = re.compile(r"\b[PI]-\d{6,}\b") _NUMBER_RE = re.compile(r"(? set[str]: - """原文/事件中的数字集合（SAG grounding 的确定性口径）。""" + """Return numeric strings using the deterministic SAG grounding policy.""" return set(_NUMBER_RE.findall(text or "")) -def event_anchor(content: str) -> str | None: - """事件的完整项目编号锚点；取首个出现的编号。""" - match = PROJECT_ID_RE.search(content or "") - return match.group(0) if match else None +def event_anchor(content: str, pattern: str | None = None) -> str | None: + """Return the first configured anchor match, if any.""" + if pattern is None: + return None + match = re.compile(pattern).search(content or "") + if match is None: + return None + if match.lastindex: + return match.group(1) + return match.group(0) -def event_node_name(anchor: str, title: str, content: str) -> str: - """确定性事件节点 id：event:{anchor}:{hash12}（内容 SHA，重叠 chunk 天然去重）。""" - digest = hashlib.sha256(f"{anchor}\n{title}\n{content}".encode("utf-8")).hexdigest()[:12] - return f"event:{anchor}:{digest}" +def event_node_name(anchor: str | None, title: str, content: str, *, legacy: bool = True) -> str: + """Create a deterministic node ID; legacy PMS mode preserves persisted IDs.""" + if legacy: + digest = hashlib.sha256(f"{anchor}\n{title}\n{content}".encode("utf-8")).hexdigest()[:12] + return f"event:{anchor}:{digest}" + digest = hashlib.sha256( + json.dumps([anchor, title, content], ensure_ascii=False, separators=(",", ":")).encode("utf-8") + ).hexdigest()[:20] + return f"event:{digest}" -def parse_extraction_response(response: str, *, chunk_content: str) -> tuple[list[dict], dict[str, int]]: - """解析并校验 LLM 事件抽取响应，返回 (有效事件列表, 丢弃计数)。 +def parse_extraction_response( + response: str, + *, + chunk_content: str, + entity_types: list[dict] | None = None, + anchor_pattern: str | None = None, + anchor_required: bool | None = None, + number_grounding: bool = True, +) -> tuple[list[dict], dict[str, int]]: + """Parse the strict event response contract under an explicit profile policy. - 严格合同（SAG schema.py 口径）：顶层/事件/实体禁合同外字段；is_valid=False、 - 无项目锚点、数字不接地、实体类型越界的事件整条丢弃并计数。 + Omitted policy arguments retain the historical PMS behavior for callers that + import this pure-stdlib helper directly. """ dropped: dict[str, int] = defaultdict(int) text = (response or "").strip() @@ -59,22 +77,32 @@ def parse_extraction_response(response: str, *, chunk_content: str) -> tuple[lis except json.JSONDecodeError: dropped["response_not_json"] += 1 return [], dict(dropped) - if not isinstance(payload, dict) or set(payload.keys()) != {"type", "data"}: + if not isinstance(payload, dict) or set(payload) != {"type", "data"}: dropped["response_contract_invalid"] += 1 return [], dict(dropped) data = payload.get("data") - if not isinstance(data, dict) or set(data.keys()) != {"items"}: + if not isinstance(data, dict) or set(data) != {"items"} or not isinstance(data["items"], list): dropped["response_contract_invalid"] += 1 return [], dict(dropped) - evidence_numbers = numbers_in(chunk_content) + profile_policy = entity_types is not None or anchor_pattern is not None or anchor_required is not None + if profile_policy: + types = entity_types or [] + allowed_types = {str(item["type"]).strip().upper() for item in types} + if anchor_required is None: + anchor_required = True + else: + types = PMS_ENTITY_TYPES + allowed_types = {item["type"] for item in types} + anchor_pattern = PROJECT_ID_RE.pattern + anchor_required = True + evidence_numbers = numbers_in(chunk_content) if number_grounding else set() valid_events: list[dict] = [] - for item in data.get("items") or []: + for item in data["items"]: if not isinstance(item, dict): dropped["event_not_object"] += 1 continue - extra_keys = set(item.keys()) - _ALLOWED_EVENT_KEYS - if extra_keys: + if set(item) - _ALLOWED_EVENT_KEYS: dropped["event_extra_fields"] += 1 continue title = str(item.get("title") or "").strip() @@ -86,20 +114,22 @@ def parse_extraction_response(response: str, *, chunk_content: str) -> tuple[lis if item.get("is_valid") is False: dropped["event_self_invalid"] += 1 continue - anchor = event_anchor(content) - if anchor is None: + anchor = event_anchor(content, anchor_pattern) + if anchor_required and anchor is None: dropped["event_missing_anchor"] += 1 continue - ungrounded = {n for n in numbers_in(f"{title} {content}") if n not in evidence_numbers} - if ungrounded: + if number_grounding and {n for n in numbers_in(f"{title} {content}") if n not in evidence_numbers}: dropped["event_number_not_grounded"] += 1 continue entities: list[dict] = [] + if not isinstance(entities_raw, list): + dropped["event_entities_invalid"] += 1 + continue for entity in entities_raw: if not isinstance(entity, dict): dropped["entity_not_object"] += 1 continue - if set(entity.keys()) - _ALLOWED_ENTITY_KEYS: + if set(entity) - _ALLOWED_ENTITY_KEYS: dropped["entity_extra_fields"] += 1 continue etype = str(entity.get("type") or "").strip().upper() @@ -108,12 +138,9 @@ def parse_extraction_response(response: str, *, chunk_content: str) -> tuple[lis if not etype or not ename or not edesc: dropped["entity_blank"] += 1 continue - if etype not in _ALLOWED_TYPES: + if etype not in allowed_types: dropped["entity_type_unknown"] += 1 continue entities.append({"type": etype, "name": ename, "description": edesc}) - valid_events.append( - {"title": title, "content": content, "anchor": anchor, "entities": entities} - ) + valid_events.append({"title": title, "content": content, "anchor": anchor, "entities": entities}) return valid_events, dict(dropped) - diff --git a/graphgen/models/kg_builder/event_entity_kg_builder.py b/graphgen/models/kg_builder/event_entity_kg_builder.py index 1aa44489..f7dfee7f 100644 --- a/graphgen/models/kg_builder/event_entity_kg_builder.py +++ b/graphgen/models/kg_builder/event_entity_kg_builder.py @@ -1,54 +1,33 @@ -"""Event-entity KG builder (PMS fork, SAG-adapted). +"""Profile-driven SAG-adapted event/entity graph builder. -来源与许可：核心设计改编自 Zleap-AI/SAG `zleap/sag/modules/extract/{schema,grounding}.py` -与 `prompts/extract_document.yaml` v3.1（MIT，Copyright (c) 2026 sag contributors， -arXiv:2606.15971）。改编点见 PMS fork docs/PMS_PATCHES.md #7 与 Phase 2 方案文档 §3.2。 - -与 LightRAGKGBuilder 的差异： -- 每 chunk 抽取"语义完整的事件"而非三元组：事件 = EVENT 节点 + "事件→参与者" - 星型边（超边的星型编码，n 元关系不拆三元组）； -- 严格响应合同（禁合同外字段、is_valid 自报、无锚 fail-closed 丢弃）； -- 数字接地（SAG grounding 移植）：事件中一切数字必须在原文逐字出现，否则弃； -- 事件节点 id 确定性（内容 SHA），chunk 重叠去重由官方 merge_nodes SEP 合并免费获得。 - -extract() 返回与官方 ({entity_name: [node]}, {(src, tgt): [edge]}) 相同的形状， -merge_nodes / merge_edges / 描述摘要全部复用官方实现。 +Domain policy (prompt, entity types, anchor rules and grounding) comes from a +validated event extraction profile. Generic graph construction remains shared. +SAG-derived adaptation and license notes are recorded in docs/PMS_PATCHES.md. """ from __future__ import annotations import json - from collections import defaultdict -from typing import Any, Dict, List, Tuple +from typing import Dict, List, Tuple from graphgen.bases import BaseGraphStorage, BaseLLMWrapper, Chunk -from graphgen.templates import EVENT_ENTITY_EXTRACTION_PROMPT -from graphgen.templates.kg.event_entity_extraction import PMS_ENTITY_TYPES -from graphgen.utils import detect_main_language, logger +from graphgen.models.kg_builder.event_contract import event_node_name, parse_extraction_response +from graphgen.templates.event_profile_loader import EventExtractionProfile, load_event_profile +from graphgen.utils import logger _EVENT_DESC_CAP = 4000 _ENTITY_DESC_CAP = 2000 -from graphgen.models.kg_builder.event_contract import ( - PROJECT_ID_RE, - _ALLOWED_ENTITY_KEYS, - _ALLOWED_EVENT_KEYS, - _ALLOWED_TYPES, - event_anchor, - event_node_name, - numbers_in, - parse_extraction_response, -) - class EventEntityKGBuilder: """事件-实体抽取器：extract → 星型编码节点/边；合并复用官方 LightRAG merge。""" - def __init__(self, llm_client: BaseLLMWrapper): + def __init__(self, llm_client: BaseLLMWrapper, profile: str | EventExtractionProfile | None = None): from graphgen.models import LightRAGKGBuilder self.llm_client = llm_client + self.profile = load_event_profile(profile) if isinstance(profile, (str, type(None))) else profile self._merger = LightRAGKGBuilder(llm_client=llm_client) async def extract( @@ -56,8 +35,7 @@ async def extract( ) -> Tuple[Dict[str, List[dict]], Dict[Tuple[str, str], List[dict]]]: """单 chunk 事件抽取，返回与官方同形的 (nodes_data, edges_data)。""" content = chunk.content - language = detect_main_language(content) - entity_types_json = json.dumps(PMS_ENTITY_TYPES, ensure_ascii=False) + entity_types_json = json.dumps(list(self.profile.entity_types), ensure_ascii=False) request_json = json.dumps( { "type": "request", @@ -68,11 +46,17 @@ async def extract( }, ensure_ascii=False, ).replace("__ENTITY_TYPES__", entity_types_json) - # 模板含 JSON 字面大括号，禁用 str.format；用占位符替换注入输入。 - prompt = EVENT_ENTITY_EXTRACTION_PROMPT[language].replace("{input_text}", request_json) + prompt = self.profile.prompt.replace("{input_text}", request_json) response = await self.llm_client.generate_answer(prompt) - events, dropped = parse_extraction_response(response, chunk_content=content) + events, dropped = parse_extraction_response( + response, + chunk_content=content, + entity_types=list(self.profile.entity_types), + anchor_pattern=self.profile.anchor_pattern, + anchor_required=self.profile.anchor_required, + number_grounding=self.profile.number_grounding, + ) if sum(dropped.values()): logger.warning( "Event extraction dropped: %s (chunk=%s)", dropped, chunk.id @@ -84,30 +68,34 @@ async def extract( edges: Dict[Tuple[str, str], List[dict]] = defaultdict(list) for event in events: anchor = event["anchor"] - event_name = event_node_name(anchor, event["title"], event["content"]) + event_name = event_node_name( + anchor, event["title"], event["content"], legacy=self.profile.legacy_event_id + ) description = f"{event['title']}：{event['content']}" - # 事件节点（EVENT 实体，扁平属性；参与者由星型边表达） + # 事件节点使用通用节点属性；分区器读取 anchor，不解析节点 ID。 nodes[event_name].append( { - "entity_type": "EVENT", + "entity_type": self.profile.event_entity_type, "entity_name": event_name, + "graph_role": "event", "description": description, + "anchor": anchor or "", "source_id": chunk.id, } ) - # 参与者实体（去重保序；锚点项目实体若缺失则补齐） + # Add the configured anchor entity only when the profile requests one. seen: set[str] = set() participants: list[dict] = [] for entity in event["entities"]: if entity["name"] not in seen: seen.add(entity["name"]) participants.append(entity) - if anchor not in seen: + if anchor is not None and self.profile.anchor_entity_type and anchor not in seen: participants.append( { - "type": "PROJECT", + "type": self.profile.anchor_entity_type, "name": anchor, - "description": f"项目 {anchor}，本事项的锚定项目", + "description": f"锚点 {anchor}，本事项的分组标识", } ) seen.add(anchor) @@ -133,7 +121,7 @@ async def extract( async def merge_nodes(self, node_data: tuple, kg_instance: BaseGraphStorage) -> dict: entity_name, node_list = node_data - if node_list and str(node_list[0].get("entity_type", "")).upper() == "EVENT": + if node_list and str(node_list[0].get("entity_type", "")).upper() == self.profile.event_entity_type: return await self._merge_event_node(entity_name, node_list, kg_instance) # 星型事件参与者是图投影，不需要再次 LLM 摘要；只做确定性去重合并， # 避免 LightRAG summarizer 把跨事件上下文扩写成伪业务事实。 @@ -166,13 +154,20 @@ async def _merge_event_node( ) -> dict: """事件节点确定性合并：SEP 去重拼接（截断），不走 LLM 摘要—— 官方摘要器会把事件 id 当实体名写进描述，污染出题上下文。""" + existing = kg_instance.get_node(entity_name) or {} + anchors = {str(dp.get("anchor") or "") for dp in node_list if dp.get("anchor")} + existing_anchor = str(existing.get("anchor") or "") + if existing_anchor: + anchors.add(existing_anchor) merged = "".join(sorted({dp["description"] for dp in node_list})) if len(merged) > _EVENT_DESC_CAP: merged = merged[:_EVENT_DESC_CAP] node_data_dict = { - "entity_type": "EVENT", + "entity_type": self.profile.event_entity_type, "entity_name": entity_name, + "graph_role": "event", "description": merged, + "anchor": "".join(sorted(anchors)), "source_id": "".join( sorted({sid for dp in node_list for sid in str(dp.get("source_id", "")).split("") if sid} | {sid for sid in str((kg_instance.get_node(entity_name) or {}).get("source_id", "")).split("") if sid}) @@ -184,23 +179,31 @@ async def _merge_event_node( async def merge_edges(self, edges_data: tuple, kg_instance: BaseGraphStorage) -> dict: (src_id, tgt_id), edge_list = edges_data - if src_id.startswith("event:") or tgt_id.startswith("event:"): - # 事件成员边同样确定性合并，不做 LLM 摘要 + event_ids = { + str(dp["src_id"]) + for dp in edge_list + if dp.get("event_member") and dp.get("src_id") + } + if src_id in event_ids or tgt_id in event_ids: + # 事件成员边同样确定性合并，不做 LLM 摘要。 + event_id = src_id if src_id in event_ids else tgt_id + participant_id = tgt_id if event_id == src_id else src_id merged = "".join(sorted({dp["description"] for dp in edge_list})) if len(merged) > _EVENT_DESC_CAP: merged = merged[:_EVENT_DESC_CAP] + existing_edge = kg_instance.get_edge(src_id, tgt_id) or kg_instance.get_edge(tgt_id, src_id) or {} source_ids = { sid for dp in edge_list for sid in str(dp.get("source_id", "")).split("") if sid } | { - sid for sid in str((kg_instance.get_edge(src_id, tgt_id) or {}).get("source_id", "")).split("") if sid + sid for sid in str(existing_edge.get("source_id", "")).split("") if sid } edge_data = { - "src_id": src_id, - "tgt_id": tgt_id, + "src_id": event_id, + "tgt_id": participant_id, "description": merged, "source_id": "".join(sorted(source_ids)), "length": self._merger.tokenizer.count_tokens(merged), } - kg_instance.upsert_edge(src_id, tgt_id, edge_data=edge_data) + kg_instance.upsert_edge(event_id, participant_id, edge_data=edge_data) return edge_data return await self._merger.merge_edges(edges_data, kg_instance=kg_instance) diff --git a/graphgen/models/partitioner/event_join_partitioner.py b/graphgen/models/partitioner/event_join_partitioner.py index 3414ff16..327e447d 100644 --- a/graphgen/models/partitioner/event_join_partitioner.py +++ b/graphgen/models/partitioner/event_join_partitioner.py @@ -1,27 +1,15 @@ -"""Event-join partitioner (PMS fork, SAG join semantics). +"""Generic shared-participant event join partitioner (SAG join semantics). -来源：共享实体 join 语义翻译自 Zleap-AI/SAG `modules/search/base.py` 的 -entity→event→relation 两跳查询（MIT，arXiv:2606.15971）；离线物化见 Phase 2 方案文档。 - -事件 = EVENT 节点（id 形如 ``event:{anchor}:{hash12}``）+ 星型成员边。分区算法： -1. 种子事件按节点 id 排序（确定性）； -2. 通过共享参与者实体 join 一跳邻居事件； -3. 硬约束：只接受同 project_anchor 事件（簇项目纯度 100%）； -4. 容量上限：max_events_per_community 或事件描述 token 总和。 - -产出官方 ``Community``（nodes = 事件+参与者实体，edges = 星型成员边）， -metadata 携带 event_ids 供出题端 support 校验。 +Anchor isolation reads an explicit node attribute and never infers domain data +from event ID strings. """ -import math from collections import deque from typing import Any, Iterable, List, Optional, Set, Tuple from graphgen.bases import BaseGraphStorage, BasePartitioner from graphgen.bases.datatypes import Community -EVENT_NODE_PREFIX = "event:" - class EventJoinPartitioner(BasePartitioner): """EVENT 星型子图 → 同项目事件簇（官方 Community 数据类型）。""" @@ -32,13 +20,24 @@ def partition( max_events_per_community: int = 6, max_tokens_per_community: int = 4096, min_events_per_community: int = 1, + event_entity_type: str = "EVENT", + anchor_attribute: str = "anchor", + require_same_anchor: bool = True, **kwargs: Any, ) -> Iterable[Community]: + if not isinstance(require_same_anchor, bool): + raise ValueError("event_join_require_same_anchor_must_be_bool") + if not isinstance(event_entity_type, str) or not event_entity_type.strip(): + raise ValueError("event_join_event_entity_type_required") + if not isinstance(anchor_attribute, str) or not anchor_attribute.strip(): + raise ValueError("event_join_anchor_attribute_required") + if max_events_per_community < 1 or min_events_per_community < 1 or max_tokens_per_community < 1: + raise ValueError("event_join_limits_must_be_positive") events: List[Tuple[str, dict]] = [] node_dict: dict[str, dict] = {} for nid, data in g.get_all_nodes(): node_dict[nid] = data - if str(data.get("entity_type", "")).upper() == "EVENT": + if str(data.get("entity_type", "")).upper() == event_entity_type.upper(): events.append((nid, data)) events.sort(key=lambda item: item[0]) @@ -54,8 +53,7 @@ def partition( used: Set[str] = set() def _anchor(event_id: str) -> str: - # event:{anchor}:{hash} - return event_id[len(EVENT_NODE_PREFIX):].rsplit(":", 1)[0] + return str(node_dict[event_id].get(anchor_attribute, "")) def _grow(seed: Tuple[str, dict]) -> Optional[Community]: member_events: List[str] = [] @@ -64,6 +62,8 @@ def _grow(seed: Tuple[str, dict]) -> Optional[Community]: member_edges: Set[frozenset] = set() token_sum = 0 anchor = _anchor(seed[0]) + if require_same_anchor and not anchor: + raise ValueError(f"event_join_anchor_missing:{seed[0]}") queue = deque([seed[0]]) while queue: @@ -87,7 +87,7 @@ def _grow(seed: Tuple[str, dict]) -> Optional[Community]: for other in events_by_entity.get(entity_id, []): if other in used or other in event_set: continue - if _anchor(other) != anchor: + if require_same_anchor and _anchor(other) != anchor: continue if token_sum + int(node_dict[other].get("length", 0)) > max_tokens_per_community: continue diff --git a/graphgen/operators/build_kg/build_event_kg.py b/graphgen/operators/build_kg/build_event_kg.py index a25ca139..95c42236 100644 --- a/graphgen/operators/build_kg/build_event_kg.py +++ b/graphgen/operators/build_kg/build_event_kg.py @@ -1,4 +1,4 @@ -"""Event-entity KG construction over text chunks (PMS fork).""" +"""Construct an event/entity star graph from text chunks.""" from __future__ import annotations @@ -16,9 +16,10 @@ def build_event_entity_kg( llm_client: BaseLLMWrapper, kg_instance: BaseGraphStorage, chunks: List[Chunk], + event_profile: str | None = None, ) -> tuple: """chunks → 事件星型图（合并/摘要/SEP 血缘全部复用官方 LightRAG merge）。""" - kg_builder = EventEntityKGBuilder(llm_client=llm_client) + kg_builder = EventEntityKGBuilder(llm_client=llm_client, profile=event_profile) results = run_concurrent( kg_builder.extract, @@ -30,11 +31,21 @@ def build_event_entity_kg( nodes = defaultdict(list) edges = defaultdict(list) + event_ids = { + event_id + for node_map, _ in results + for event_id, rows in node_map.items() + if rows and str(rows[0].get("entity_type", "")).upper() == kg_builder.profile.event_entity_type + } for n, e in results: for k, v in n.items(): nodes[k].extend(v) for k, v in e.items(): - edges[tuple(sorted(k))].extend(v) + normalized_key = tuple(sorted(k)) + if any(endpoint in event_ids for endpoint in normalized_key): + for edge in v: + edge["event_member"] = True + edges[normalized_key].extend(v) nodes = run_concurrent( lambda kv: kg_builder.merge_nodes(kv, kg_instance=kg_instance), diff --git a/graphgen/operators/build_kg/build_kg_service.py b/graphgen/operators/build_kg/build_kg_service.py index 9a3c484c..c86c8657 100644 --- a/graphgen/operators/build_kg/build_kg_service.py +++ b/graphgen/operators/build_kg/build_kg_service.py @@ -28,9 +28,13 @@ def __init__( ) self.build_kwargs = build_kwargs self.max_loop: int = int(self.build_kwargs.get("max_loop", 3)) - # PMS fork（Phase 2）：文本 KG 抽取方法分发。默认 light_rag 与官方逐字节 - # 一致；event_entity 走事件-实体星型编码（见 models/kg_builder/event_entity_kg_builder.py）。 + # Opt-in fork capability. Existing configs continue to use LightRAG by default. self.kg_method: str = str(self.build_kwargs.get("kg_method", "light_rag")) + self.event_profile = self.build_kwargs.get("event_profile") + if self.kg_method == "event_entity": + from graphgen.templates.event_profile_loader import load_event_profile + + self.event_profile = load_event_profile(self.event_profile).profile_id def process(self, batch: list) -> Tuple[list, dict]: """ @@ -59,6 +63,7 @@ def process(self, batch: list) -> Tuple[list, dict]: llm_client=self.llm_client, kg_instance=self.graph_storage, chunks=text_chunks, + event_profile=self.event_profile, ) nodes += text_nodes edges += text_edges diff --git a/graphgen/operators/generate/generate_service.py b/graphgen/operators/generate/generate_service.py index e6676fe6..5f35f2ab 100644 --- a/graphgen/operators/generate/generate_service.py +++ b/graphgen/operators/generate/generate_service.py @@ -122,7 +122,13 @@ def process(self, batch: list) -> Tuple[list, dict]: Generate question-answer pairs based on nodes and edges. """ logger.info("[Generation] mode: %s, batches: %d", self.method, len(batch)) - triples = [(item["nodes"], item["edges"]) for item in batch] + from graphgen.models.generator.support import build_generation_view, validate_support + + generation_views = [build_generation_view(item) for item in batch] + triples = [ + (view["nodes"], view["edges"]) + for view in generation_views + ] results = run_concurrent( self.generator.generate, triples, @@ -134,19 +140,13 @@ def process(self, batch: list) -> Tuple[list, dict]: meta_updates = {} final_results = [] - for item, qa_pairs in zip(batch, results): + for item, view, qa_pairs in zip(batch, generation_views, results): if not qa_pairs: continue - # PMS fork（Phase 2 升级 C）：.cited 必须全部存在于本分区 - # 节点集合，否则整块剥离、题面保留（题面质量与审计质量解耦）。 - node_names = { - node[0] - for node in item.get("nodes", []) - if isinstance(node, (list, tuple)) and node - } for qa_pair in qa_pairs: support = qa_pair.pop("support", None) if isinstance(qa_pair, dict) else None - support = validate_support(support, node_names) + support = view["resolve_support"](support) + support = validate_support(support, set(view["canonical_nodes"])) if support: qa_pair["support"] = support res = self.generator.format_generation_results( diff --git a/graphgen/operators/partition/partition_service.py b/graphgen/operators/partition/partition_service.py index db355769..cd1c1dca 100644 --- a/graphgen/operators/partition/partition_service.py +++ b/graphgen/operators/partition/partition_service.py @@ -29,6 +29,11 @@ def __init__( self.tokenizer_instance: BaseTokenizer = Tokenizer(model_name=tokenizer_model) method = partition_kwargs["method"] self.method_params = partition_kwargs.get("method_params", {}) + if method == "event_join": + self.method_params = dict(self.method_params) + self.method_params.setdefault("event_entity_type", partition_kwargs.get("event_entity_type", "EVENT")) + self.method_params.setdefault("anchor_attribute", partition_kwargs.get("anchor_attribute", "anchor")) + self.method_params.setdefault("require_same_anchor", partition_kwargs.get("require_same_anchor", True)) if method == "bfs": from graphgen.models import BFSPartitioner diff --git a/graphgen/templates/event_profile_loader.py b/graphgen/templates/event_profile_loader.py new file mode 100644 index 00000000..c5a502a3 --- /dev/null +++ b/graphgen/templates/event_profile_loader.py @@ -0,0 +1,129 @@ +"""Load and validate packaged event extraction profiles.""" +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass +from importlib.resources import files +from pathlib import Path +from typing import Any + +PROFILE_SCHEMA = "graphgen_event_profile.v1" +PROFILE_ROOT = files("graphgen.templates").joinpath("event_profiles") + + +@dataclass(frozen=True) +class EventExtractionProfile: + profile_id: str + entity_types: tuple[dict[str, str], ...] + prompt: str + anchor_pattern: str | None + anchor_required: bool + number_grounding: bool + event_entity_type: str + anchor_entity_type: str | None + legacy_event_id: bool + manifest_sha256: str + prompt_sha256: str + + +def _load_prompt_resource(profile_id: str, prompt_file: str) -> bytes: + if not prompt_file or Path(prompt_file).name != prompt_file: + raise ValueError("event_profile_prompt_file_invalid") + return PROFILE_ROOT.joinpath(profile_id, prompt_file).read_bytes() + + +def load_event_profile(profile: str | None = None) -> EventExtractionProfile: + """Load a built-in profile ID or explicit manifest path; fail closed.""" + profile_id = "pms_event.v1" if profile is None else profile + if not isinstance(profile_id, str) or not profile_id.strip(): + raise ValueError("event_profile_must_be_non_empty_string") + if not "/" in profile_id and not "\\" in profile_id and not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}", profile_id): + raise ValueError("event_profile_id_invalid") + if "/" in profile_id or "\\" in profile_id: + manifest_path = Path(profile_id).resolve() + if not manifest_path.is_file(): + raise ValueError(f"event_profile_not_found:{profile_id}") + manifest_bytes = manifest_path.read_bytes() + data = json.loads(manifest_bytes.decode("utf-8")) + prompt_file = str(data.get("prompt_file", "")) + if not prompt_file or Path(prompt_file).name != prompt_file: + raise ValueError("event_profile_prompt_file_invalid") + prompt_path = (manifest_path.parent / prompt_file).resolve() + if prompt_path.parent != manifest_path.parent: + raise ValueError("event_profile_prompt_file_invalid") + prompt_bytes = prompt_path.read_bytes() + else: + manifest_resource = PROFILE_ROOT.joinpath(profile_id, "profile.json") + if not manifest_resource.is_file(): + raise ValueError(f"event_profile_not_found:{profile_id}") + manifest_bytes = manifest_resource.read_bytes() + data = json.loads(manifest_bytes.decode("utf-8")) + prompt_bytes = _load_prompt_resource(profile_id, str(data.get("prompt_file", ""))) + + if not isinstance(data, dict) or data.get("schema_version") != PROFILE_SCHEMA: + raise ValueError("event_profile_schema_unsupported") + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}", str(data.get("profile_id", ""))): + raise ValueError("event_profile_id_invalid") + if data.get("profile_id") != profile_id and "/" not in profile_id and "\\" not in profile_id: + raise ValueError("event_profile_id_mismatch") + entity_types = data.get("entity_types") + if not isinstance(entity_types, list) or not entity_types: + raise ValueError("event_profile_entity_types_required") + normalized: list[dict[str, str]] = [] + seen: set[str] = set() + for item in entity_types: + if not isinstance(item, dict) or set(item) != {"type", "description"}: + raise ValueError("event_profile_entity_type_invalid") + name = str(item["type"]).strip().upper() + description = str(item["description"]).strip() + if not re.fullmatch(r"[A-Z][A-Z0-9_]{0,63}", name) or not description or name in seen: + raise ValueError("event_profile_entity_type_invalid") + seen.add(name) + normalized.append({"type": name, "description": description}) + + anchor = data.get("anchor") + if not isinstance(anchor, dict) or set(anchor) != {"required", "pattern", "entity_type"}: + raise ValueError("event_profile_anchor_invalid") + required, pattern, anchor_entity_type = anchor["required"], anchor["pattern"], anchor["entity_type"] + if not isinstance(required, bool) or (pattern is not None and not isinstance(pattern, str)): + raise ValueError("event_profile_anchor_invalid") + if required and not pattern: + raise ValueError("event_profile_required_anchor_pattern_missing") + if pattern: + try: + re.compile(pattern) + except re.error as error: + raise ValueError("event_profile_anchor_pattern_invalid") from error + if anchor_entity_type is not None and (not isinstance(anchor_entity_type, str) or anchor_entity_type.upper() not in seen): + raise ValueError("event_profile_anchor_entity_type_unknown") + if isinstance(anchor_entity_type, str): + anchor_entity_type = anchor_entity_type.upper() + + grounding = data.get("grounding") + if not isinstance(grounding, dict) or set(grounding) != {"number_grounding"} or not isinstance(grounding["number_grounding"], bool): + raise ValueError("event_profile_grounding_invalid") + graph = data.get("graph") + if not isinstance(graph, dict) or set(graph) != {"event_entity_type", "legacy_event_id"}: + raise ValueError("event_profile_graph_invalid") + event_entity_type = str(graph["event_entity_type"]).strip().upper() + if not re.fullmatch(r"[A-Z][A-Z0-9_]{0,63}", event_entity_type) or not isinstance(graph["legacy_event_id"], bool): + raise ValueError("event_profile_event_entity_type_invalid") + prompt = prompt_bytes.decode("utf-8") + if not prompt.strip() or prompt.count("{input_text}") != 1: + raise ValueError("event_profile_prompt_invalid") + + return EventExtractionProfile( + profile_id=str(data["profile_id"]), + entity_types=tuple(normalized), + prompt=prompt, + anchor_pattern=pattern, + anchor_required=required, + number_grounding=grounding["number_grounding"], + event_entity_type=event_entity_type, + anchor_entity_type=anchor_entity_type, + legacy_event_id=graph["legacy_event_id"], + manifest_sha256=hashlib.sha256(manifest_bytes).hexdigest(), + prompt_sha256=hashlib.sha256(prompt_bytes).hexdigest(), + ) diff --git a/graphgen/templates/event_profiles/example_event.v1/profile.json b/graphgen/templates/event_profiles/example_event.v1/profile.json new file mode 100644 index 00000000..95fe1403 --- /dev/null +++ b/graphgen/templates/event_profiles/example_event.v1/profile.json @@ -0,0 +1,13 @@ +{ + "schema_version": "graphgen_event_profile.v1", + "profile_id": "example_event.v1", + "prompt_file": "prompt.zh.txt", + "entity_types": [ + {"type": "MISSION", "description": "一次需要完成的任务或行动"}, + {"type": "ACTOR", "description": "参与任务的个人或组织"}, + {"type": "RESOURCE", "description": "任务使用或影响的资源"} + ], + "anchor": {"required": true, "pattern": "\\bCASE-[A-Z0-9-]+\\b", "entity_type": null}, + "grounding": {"number_grounding": true}, + "graph": {"event_entity_type": "EVENT", "legacy_event_id": false} +} diff --git a/graphgen/templates/event_profiles/example_event.v1/prompt.zh.txt b/graphgen/templates/event_profiles/example_event.v1/prompt.zh.txt new file mode 100644 index 00000000..5d6ec7c2 --- /dev/null +++ b/graphgen/templates/event_profiles/example_event.v1/prompt.zh.txt @@ -0,0 +1,28 @@ +## Role + +你是一个通用的事件与参与实体抽取器。只依据当前文本片段，识别可独立陈述的事件及其参与者。 + +## Rules + +- 每个事件以完整、可读的事实句表达，区分已经发生、正在发生、计划和推测。 +- 不补充原文未支持的常识、背景或因果结论；信息不足时不输出该事件。 +- 只使用输入 meta.entity_types 中声明的实体类型，不自造类型。 +- 实体 description 说明其在该事件中的作用；并列参与者分别列出。 +- 如果输入文本存在 profile 配置的锚点，事件 content 应保留该锚点；不得自行创造锚点。 +- 所有数字必须可在当前输入原文中逐字找到。 +- 相对时间保留原文，不自行换算。 + +## Input + +输入 JSON：{"type":"request","data":{"items":[{"id":1,"content":"..."}],"meta":{"entity_types":[{"type":"...","description":"..."}]}}} + +## Output + +只返回严格 JSON，不得包含 Markdown 或其他文字： +{"type":"response","data":{"items":[{"title":"...","content":"...","entities":[{"type":"...","name":"...","description":"..."}],"is_valid":true}]}} +无有效事件时返回：{"type":"response","data":{"items":[]}} + +给定输入： +{input_text} + +输出： diff --git a/graphgen/templates/event_profiles/generic_event.v1/profile.json b/graphgen/templates/event_profiles/generic_event.v1/profile.json new file mode 100644 index 00000000..b6155c66 --- /dev/null +++ b/graphgen/templates/event_profiles/generic_event.v1/profile.json @@ -0,0 +1,12 @@ +{ + "schema_version": "graphgen_event_profile.v1", + "profile_id": "generic_event.v1", + "prompt_file": "prompt.zh.txt", + "entity_types": [ + {"type": "ACTION", "description": "An action or occurrence"}, + {"type": "ACTOR", "description": "A participant in the event"} + ], + "anchor": {"required": false, "pattern": null, "entity_type": null}, + "grounding": {"number_grounding": true}, + "graph": {"event_entity_type": "EVENT", "legacy_event_id": false} +} diff --git a/graphgen/templates/event_profiles/generic_event.v1/prompt.zh.txt b/graphgen/templates/event_profiles/generic_event.v1/prompt.zh.txt new file mode 100644 index 00000000..b7d9b189 --- /dev/null +++ b/graphgen/templates/event_profiles/generic_event.v1/prompt.zh.txt @@ -0,0 +1,27 @@ +## Role + +You extract events and participants from the supplied text. Report only facts supported by the input. + +## Rules + +- Write each event as a complete, independently readable factual statement. +- Distinguish completed, ongoing, planned, and uncertain actions. +- Use only entity types listed in `meta.entity_types`; do not invent types. +- Describe each participant's role in the event and split distinct participants. +- Preserve numbers and dates only when supported by the source. Keep relative dates as written. +- Return no event when the source does not support a useful standalone fact. + +## Input + +The input is JSON with `type=request`, `data.items` and `data.meta.entity_types`. + +## Output + +Return only JSON matching this exact shape: +{"type":"response","data":{"items":[{"title":"...","content":"...","entities":[{"type":"...","name":"...","description":"..."}],"is_valid":true}]}} +For no events return `{"type":"response","data":{"items":[]}}`. + +Input: +{input_text} + +Output: diff --git a/graphgen/templates/event_profiles/pms_event.v1/profile.json b/graphgen/templates/event_profiles/pms_event.v1/profile.json new file mode 100644 index 00000000..5a39839e --- /dev/null +++ b/graphgen/templates/event_profiles/pms_event.v1/profile.json @@ -0,0 +1,18 @@ +{ + "schema_version": "graphgen_event_profile.v1", + "profile_id": "pms_event.v1", + "prompt_file": "prompt.zh.txt", + "entity_types": [ + {"type": "PROJECT", "description": "项目本体，含完整项目编号"}, + {"type": "PERSON", "description": "人员，项目成员或干系人"}, + {"type": "ROLE", "description": "项目内职务或角色"}, + {"type": "ORGANIZATION", "description": "组织、部门、公司或团队"}, + {"type": "PHASE", "description": "项目阶段或状态"}, + {"type": "LOCATION", "description": "地点或区域"}, + {"type": "DATE", "description": "日期或时间"}, + {"type": "METRIC", "description": "数量、比例等指标"} + ], + "anchor": {"required": true, "pattern": "\\b[PI]-\\d{6,}\\b", "entity_type": "PROJECT"}, + "grounding": {"number_grounding": true}, + "graph": {"event_entity_type": "EVENT", "legacy_event_id": true} +} diff --git a/graphgen/templates/event_profiles/pms_event.v1/prompt.zh.txt b/graphgen/templates/event_profiles/pms_event.v1/prompt.zh.txt new file mode 100644 index 00000000..7ae0749f --- /dev/null +++ b/graphgen/templates/event_profiles/pms_event.v1/prompt.zh.txt @@ -0,0 +1,74 @@ +## Role + +你是专业的 PMS 项目管理文档事项与实体提取器。输入是当前 Chunk 片段。你的输出用于 +构建可追溯的事项—实体图谱，供后续生成"项目查询"问答题使用。 + +## 事项提取原则 + +- 按主题组织信息，用完整的主谓宾句陈述事实，禁止机械拼接原文。 +- 一个事项应围绕一个清晰主题；content 应概括"发生了什么、涉及谁、结果如何"， + 保留必要的数字和结论。 +- 可以调整语序、消解有充分依据的指代，但不得改变主语、动作性质、因果或完成状态。 +- "计划、预计、建议、可能、正在、已经完成"必须严格区分。 +- 不得把外部常识、文件名或模型推测写成原文事实。 +- 每个事项必须锚定在一个明确项目上：content 中必须出现该项目的完整项目编号 + （形如 P-24084947、I-26015101，字母+横线+至少 6 位数字）。无法锚定到完整项目 + 编号的事项不得输出。 +- 项目名称在同系统可能不唯一：提及时必须连同完整项目编号一起出现。 +- 字段锁定：只允许陈述项目编号、项目名称、项目阶段、最终用户、销售工程师、 + 项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件信息不提取。 +- 禁止反查表述：人员只作为"某项目的成员/某项目某角色由谁担任"陈述， + 不得写成"某人属于哪个部门"等组织反查事实。 + +## 实体提取原则 + +- 只提取理解事项所必需的实体；实体类型只能从输入 meta.entity_types 中选择， + 没有合适类型时不要自造类型。 +- 并列实体必须拆分，例如"甲公司和乙公司"应为两个 organization 实体。 +- description 必须描述实体在本事项中的角色、动作或关系，不能只重复实体名称。 +- 代词只有在当前片段能唯一消歧时才替换；无法确定时保留原文表达。 +- 同名实体保持一致；不同实体不得因名称相近而合并。 + +## 事实边界（fail-closed） + +- 事项的每个事实、数字、结论都必须在当前 Chunk 中有直接依据。 +- 事项 content 中出现的所有数字（含编号中的数字）必须能在原文中逐字找到； + 找不到就删掉该表述或放弃该事项。 +- 当前片段信息不足以支撑独立事项时必须返回空 items，不能靠推测拼出事项。 + +## Task + +按以下顺序执行，不能跳步： + +1. **全局扫描**：识别主体、动作、对象、时间、地点、数据、结果与不确定性； + 页眉页脚、目录编号、广告、版权声明、乱码视为噪音。 +2. **聚合或拆分判断**：优先级固定为 总结性内容聚合 > 总结后的详细展开分离 > + 普通内容按主题拆分；仅有名称或修辞差异不强行拆分。 +3. **事项构建**：title 精炼表达主题与关键结论；content 独立可读、含完整项目 + 编号锚点、禁止推理说明。 +4. **实体提取**：只从该事项陈述中提取实体，保留原文正式名称、数值单位和时间说法。 +5. **最终自检**：确认无原文之外的事实；确认每个事项含完整项目编号锚点； + 确认所有数字可在原文找到；确认输出只有统一合同允许的字段。 + 全部片段为噪音或信息不足时返回 items: []。 + +## Input + +输入 JSON：{"type": "request", "data": {"items": [{"id": 1, "content": "..."}], +"meta": {"entity_types": [{"type": "...", "description": "..."}]}}} + +## Output requirements + +- 只返回统一 JSON 合同，不要 Markdown、解释文字或合同外字段： + {"type": "response", "data": {"items": [ + {"title": "...", "content": "...", + "entities": [{"type": "...", "name": "...", "description": "..."}], + "is_valid": true} + ]}} +- 无有效事项时返回 {"type": "response", "data": {"items": []}}，不得返回 null + 或制造占位事项。 +- 相对时间（如"下月"）保留原文，不自行换算。 + +给定输入： +{input_text} + +输出： diff --git a/graphgen/templates/kg/event_entity_extraction.py b/graphgen/templates/kg/event_entity_extraction.py index e74c1be8..666f9f8d 100644 --- a/graphgen/templates/kg/event_entity_extraction.py +++ b/graphgen/templates/kg/event_entity_extraction.py @@ -1,113 +1,17 @@ -# pylint: disable=C0301 -"""事件-实体抽取提示词（SAG extract_document v3.1 改编，MIT）。 +"""Backward-compatible exports for the original PMS event extraction prompt.""" +from __future__ import annotations -来源：Zleap-AI/SAG `zleap/sag/prompts/extract_document.yaml`（arXiv 2606.15971）。 -改编点（PMS project-query 场景）： -- 事件必须含完整项目编号锚点（P-/I- + ≥6 位数字），无锚事件直接不产出； -- 字段锁定 / 反查禁止 / 越界语义禁止（与 pms_policy 出题铁律同源）; -- 数字接地：事件中一切数字必须在原文出现； -- 只输出扁平事项（不用 children 层级），便于星型编码进图。 -响应是严格 JSON 合同：禁合同外字段，无效字段/无锚/数字不接地一律丢弃并计数。 -""" - -TEMPLATE_ZH: str = """## Role - -你是专业的 PMS 项目管理文档事项与实体提取器。输入是当前 Chunk 片段。你的输出用于 -构建可追溯的事项—实体图谱，供后续生成"项目查询"问答题使用。 - -## 事项提取原则 - -- 按主题组织信息，用完整的主谓宾句陈述事实，禁止机械拼接原文。 -- 一个事项应围绕一个清晰主题；content 应概括"发生了什么、涉及谁、结果如何"， - 保留必要的数字和结论。 -- 可以调整语序、消解有充分依据的指代，但不得改变主语、动作性质、因果或完成状态。 -- "计划、预计、建议、可能、正在、已经完成"必须严格区分。 -- 不得把外部常识、文件名或模型推测写成原文事实。 -- 每个事项必须锚定在一个明确项目上：content 中必须出现该项目的完整项目编号 - （形如 P-24084947、I-26015101，字母+横线+至少 6 位数字）。无法锚定到完整项目 - 编号的事项不得输出。 -- 项目名称在同系统可能不唯一：提及时必须连同完整项目编号一起出现。 -- 字段锁定：只允许陈述项目编号、项目名称、项目阶段、最终用户、销售工程师、 - 项目成员及其角色。合同/金额/交付/付款/物料/变更/风险/会议/附件信息不提取。 -- 禁止反查表述：人员只作为"某项目的成员/某项目某角色由谁担任"陈述， - 不得写成"某人属于哪个部门"等组织反查事实。 - -## 实体提取原则 - -- 只提取理解事项所必需的实体；实体类型只能从输入 meta.entity_types 中选择， - 没有合适类型时不要自造类型。 -- 并列实体必须拆分，例如"甲公司和乙公司"应为两个 organization 实体。 -- description 必须描述实体在本事项中的角色、动作或关系，不能只重复实体名称。 -- 代词只有在当前片段能唯一消歧时才替换；无法确定时保留原文表达。 -- 同名实体保持一致；不同实体不得因名称相近而合并。 - -## 事实边界（fail-closed） - -- 事项的每个事实、数字、结论都必须在当前 Chunk 中有直接依据。 -- 事项 content 中出现的所有数字（含编号中的数字）必须能在原文中逐字找到； - 找不到就删掉该表述或放弃该事项。 -- 当前片段信息不足以支撑独立事项时必须返回空 items，不能靠推测拼出事项。 - -## Task - -按以下顺序执行，不能跳步： - -1. **全局扫描**：识别主体、动作、对象、时间、地点、数据、结果与不确定性； - 页眉页脚、目录编号、广告、版权声明、乱码视为噪音。 -2. **聚合或拆分判断**：优先级固定为 总结性内容聚合 > 总结后的详细展开分离 > - 普通内容按主题拆分；仅有名称或修辞差异不强行拆分。 -3. **事项构建**：title 精炼表达主题与关键结论；content 独立可读、含完整项目 - 编号锚点、禁止推理说明。 -4. **实体提取**：只从该事项陈述中提取实体，保留原文正式名称、数值单位和时间说法。 -5. **最终自检**：确认无原文之外的事实；确认每个事项含完整项目编号锚点； - 确认所有数字可在原文找到；确认输出只有统一合同允许的字段。 - 全部片段为噪音或信息不足时返回 items: []。 - -## Input - -输入 JSON：{"type": "request", "data": {"items": [{"id": 1, "content": "..."}], -"meta": {"entity_types": [{"type": "...", "description": "..."}]}}} - -## Output requirements - -- 只返回统一 JSON 合同，不要 Markdown、解释文字或合同外字段： - {"type": "response", "data": {"items": [ - {"title": "...", "content": "...", - "entities": [{"type": "...", "name": "...", "description": "..."}], - "is_valid": true} - ]}} -- 无有效事项时返回 {"type": "response", "data": {"items": []}}，不得返回 null - 或制造占位事项。 -- 相对时间（如"下月"）保留原文，不自行换算。 - -给定输入： -{input_text} - -输出： -""" - -#: 紧凑 JSON 请求包装（与 SAG request 合同同形；PMS 单 chunk 单 items） -INPUT_FORMAT: dict = { - "items_template": '{{"type": "request", "data": {{"items": [{{"id": 1, "content": {content!r}}}], ' - '"meta": {{"entity_types": {entity_types}}}}}}}', -} - -#: PMS 实体类型集（SAG 5W1H 类型集收敛到 project-query 可查询维度） -PMS_ENTITY_TYPES: list[dict] = [ - {"type": "PROJECT", "description": "项目本体，含完整项目编号"}, - {"type": "PERSON", "description": "人员，项目成员或干系人"}, - {"type": "ROLE", "description": "项目内职务或角色，如二级项目经理、销售工程师"}, - {"type": "ORGANIZATION", "description": "组织、部门、公司或团队"}, - {"type": "PHASE", "description": "项目阶段或状态，如售前、执行"}, - {"type": "LOCATION", "description": "地点或区域"}, - {"type": "DATE", "description": "日期或时间"}, - {"type": "METRIC", "description": "数量、比例等指标"}, -] +from graphgen.templates.event_profile_loader import load_event_profile +_pms_profile = load_event_profile("pms_event.v1") +PMS_ENTITY_TYPES: list[dict] = list(_pms_profile.entity_types) +TEMPLATE_ZH = _pms_profile.prompt EVENT_ENTITY_EXTRACTION_PROMPT = { "en": TEMPLATE_ZH, "zh": TEMPLATE_ZH, - "FORMAT": { - "entity_types": None, # 运行时注入 PMS_ENTITY_TYPES - }, + "FORMAT": {"entity_types": None}, +} +INPUT_FORMAT = { + "items_template": '{{"type": "request", "data": {{"items": [{{"id": 1, "content": {content!r}}}], ' + '"meta": {{"entity_types": {entity_types}}}}}}}', } diff --git a/requirements-dev.txt b/requirements-dev.txt index 55b033e9..e079f8a6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1 +1 @@ -pytest \ No newline at end of file +pytest diff --git a/setup.py b/setup.py index 2b30f103..bc2b362c 100644 --- a/setup.py +++ b/setup.py @@ -38,14 +38,20 @@ def read_requirements(): setup( name="graphg", version=get_version(), - url="https://github.com/open-sciencelab/GraphGen", + url="https://github.com/Leon-Algo/GraphGen", description="GraphGen: Enhancing Supervised Fine-Tuning for LLMs with Knowledge-Driven Synthetic Data Generation", long_description=readme(), long_description_content_type="text/markdown", - author="open-sciencelab", - author_email="open-sciencelab@pjlab.org.cn", + author="Leon-Algo GraphGen maintainers", + author_email="", packages=find_packages(exclude=["models"]), - package_data={"GraphGen": ["configs/*"]}, + package_data={ + "graphgen": [ + "templates/profiles/*/*.json", + "templates/event_profiles/*/*.json", + "templates/event_profiles/*/*.txt", + ], + }, include_package_data=True, install_requires=install_packages, classifiers=[ diff --git a/tests/integration_tests/models/partitioner/test_bfs_partitioner.py b/tests/integration_tests/models/partitioner/test_bfs_partitioner.py index 48558cc1..bf6840ef 100644 --- a/tests/integration_tests/models/partitioner/test_bfs_partitioner.py +++ b/tests/integration_tests/models/partitioner/test_bfs_partitioner.py @@ -1,43 +1,39 @@ import tempfile -import pytest - from graphgen.bases.datatypes import Community -from graphgen.models import BFSPartitioner, NetworkXStorage +from graphgen.models import BFSPartitioner +from graphgen.storage import NetworkXStorage -@pytest.mark.asyncio -async def test_empty_graph(): +def test_empty_graph(): with tempfile.TemporaryDirectory() as tmpdir: storage = NetworkXStorage(working_dir=tmpdir, namespace="empty") partitioner = BFSPartitioner() - communities = await partitioner.partition(storage, max_units_per_community=5) + communities = list(partitioner.partition(storage, max_units_per_community=5)) assert communities == [] -@pytest.mark.asyncio -async def test_single_node(): +def test_single_node(): nodes = [("A", {"desc": "alone"})] edges = [] with tempfile.TemporaryDirectory() as tmpdir: storage = NetworkXStorage(working_dir=tmpdir, namespace="single_node") for nid, ndata in nodes: - await storage.upsert_node(nid, ndata) + storage.upsert_node(nid, ndata) for src, tgt, edata in edges: - await storage.upsert_edge(src, tgt, edata) + storage.upsert_edge(src, tgt, edata) partitioner = BFSPartitioner() - communities: list[Community] = await partitioner.partition( + communities: list[Community] = list(partitioner.partition( storage, max_units_per_community=5 - ) + )) assert len(communities) == 1 assert communities[0].nodes == ["A"] assert communities[0].edges == [] -@pytest.mark.asyncio -async def test_small_graph(): +def test_small_graph(): """ 0 - 1 - 2 | | | @@ -59,14 +55,14 @@ async def test_small_graph(): storage = NetworkXStorage(working_dir=tmpdir, namespace="small_graph") for nid, ndata in nodes: - await storage.upsert_node(nid, ndata) + storage.upsert_node(nid, ndata) for src, tgt, edata in edges: - await storage.upsert_edge(src, tgt, edata) + storage.upsert_edge(src, tgt, edata) partitioner = BFSPartitioner() - communities: list[Community] = await partitioner.partition( + communities: list[Community] = list(partitioner.partition( storage, max_units_per_community=4 - ) + )) assert len(communities) <= 5 diff --git a/tests/integration_tests/models/partitioner/test_dfs_partitioner.py b/tests/integration_tests/models/partitioner/test_dfs_partitioner.py index 536e139a..0176910c 100644 --- a/tests/integration_tests/models/partitioner/test_dfs_partitioner.py +++ b/tests/integration_tests/models/partitioner/test_dfs_partitioner.py @@ -1,25 +1,22 @@ import tempfile -import pytest - from graphgen.bases.datatypes import Community -from graphgen.models import DFSPartitioner, NetworkXStorage +from graphgen.models import DFSPartitioner +from graphgen.storage import NetworkXStorage -@pytest.mark.asyncio -async def test_empty_graph(): +def test_empty_graph(): with tempfile.TemporaryDirectory() as tmpdir: storage = NetworkXStorage( working_dir=tmpdir, namespace="empty", ) partitioner = DFSPartitioner() - communities = await partitioner.partition(storage, max_units_per_community=5) + communities = list(partitioner.partition(storage, max_units_per_community=5)) assert communities == [] -@pytest.mark.asyncio -async def test_single_node(): +def test_single_node(): nodes = [("A", {"desc": "alone"})] edges = [] with tempfile.TemporaryDirectory() as tmpdir: @@ -29,21 +26,20 @@ async def test_single_node(): ) for nid, ndata in nodes: - await storage.upsert_node(nid, ndata) + storage.upsert_node(nid, ndata) for src, tgt, edata in edges: - await storage.upsert_edge(src, tgt, edata) + storage.upsert_edge(src, tgt, edata) partitioner = DFSPartitioner() - communities: list[Community] = await partitioner.partition( + communities: list[Community] = list(partitioner.partition( storage, max_units_per_community=5 - ) + )) assert len(communities) == 1 assert communities[0].nodes == ["A"] assert communities[0].edges == [] -@pytest.mark.asyncio -async def test_small_graph(): +def test_small_graph(): """ 0 - 1 - 2 | | | @@ -68,17 +64,17 @@ async def test_small_graph(): ) for nid, ndata in nodes: - await storage.upsert_node(nid, ndata) + storage.upsert_node(nid, ndata) for src, tgt, edata in edges: - await storage.upsert_edge(src, tgt, edata) + storage.upsert_edge(src, tgt, edata) partitioner = DFSPartitioner() - communities: list[Community] = await partitioner.partition( + communities: list[Community] = list(partitioner.partition( storage, max_units_per_community=4 - ) + )) - assert len(communities) <= 5 + assert len(communities) <= len(nodes) all_nodes = set() all_edges = set() for c in communities: diff --git a/tests/integration_tests/models/partitioner/test_ece_partitioner.py b/tests/integration_tests/models/partitioner/test_ece_partitioner.py index 93fc2ea1..2e35df97 100644 --- a/tests/integration_tests/models/partitioner/test_ece_partitioner.py +++ b/tests/integration_tests/models/partitioner/test_ece_partitioner.py @@ -1,25 +1,22 @@ import tempfile -import pytest - from graphgen.bases.datatypes import Community -from graphgen.models import ECEPartitioner, NetworkXStorage +from graphgen.models import ECEPartitioner +from graphgen.storage import NetworkXStorage -@pytest.mark.asyncio -async def test_ece_empty_graph(): +def test_ece_empty_graph(): """ECE partitioning on an empty graph should return an empty community list.""" with tempfile.TemporaryDirectory() as tmpdir: storage = NetworkXStorage(working_dir=tmpdir, namespace="empty") partitioner = ECEPartitioner() - communities = await partitioner.partition( + communities = list(partitioner.partition( storage, max_units_per_community=5, unit_sampling="random" - ) + )) assert communities == [] -@pytest.mark.asyncio -async def test_ece_single_node(): +def test_ece_single_node(): """A single node must be placed in exactly one community under any edge-sampling strategy.""" nodes = [("A", {"desc": "alone", "length": 10, "loss": 0.1})] @@ -29,19 +26,18 @@ async def test_ece_single_node(): working_dir=tmpdir, namespace=f"single_{strategy}" ) for nid, ndata in nodes: - await storage.upsert_node(nid, ndata) + storage.upsert_node(nid, ndata) partitioner = ECEPartitioner() - communities: list[Community] = await partitioner.partition( + communities: list[Community] = list(partitioner.partition( storage, max_units_per_community=5, unit_sampling=strategy - ) + )) assert len(communities) == 1 assert communities[0].nodes == ["A"] assert communities[0].edges == [] -@pytest.mark.asyncio -async def test_ece_small_graph_random(): +def test_ece_small_graph_random(): """ 2x3 grid graph: 0 — 1 — 2 @@ -63,14 +59,14 @@ async def test_ece_small_graph_random(): with tempfile.TemporaryDirectory() as tmpdir: storage = NetworkXStorage(working_dir=tmpdir, namespace="small_random") for nid, ndata in nodes: - await storage.upsert_node(nid, ndata) + storage.upsert_node(nid, ndata) for src, tgt, edata in edges: - await storage.upsert_edge(src, tgt, edata) + storage.upsert_edge(src, tgt, edata) partitioner = ECEPartitioner() - communities: list[Community] = await partitioner.partition( + communities: list[Community] = list(partitioner.partition( storage, max_units_per_community=4, unit_sampling="random" - ) + )) # Basic integrity checks all_nodes = set() @@ -83,8 +79,7 @@ async def test_ece_small_graph_random(): assert len(all_edges) == 7 -@pytest.mark.asyncio -async def test_ece_small_graph_min_loss(): +def test_ece_small_graph_min_loss(): """ Same grid graph, but using min_loss sampling. Edges with lower loss should be preferred during community expansion. @@ -105,14 +100,14 @@ async def test_ece_small_graph_min_loss(): with tempfile.TemporaryDirectory() as tmpdir: storage = NetworkXStorage(working_dir=tmpdir, namespace="small_min") for nid, ndata in nodes: - await storage.upsert_node(nid, ndata) + storage.upsert_node(nid, ndata) for src, tgt, edata in edges: - await storage.upsert_edge(src, tgt, edata) + storage.upsert_edge(src, tgt, edata) partitioner = ECEPartitioner() - communities: list[Community] = await partitioner.partition( + communities: list[Community] = list(partitioner.partition( storage, max_units_per_community=4, unit_sampling="min_loss" - ) + )) all_nodes = set() all_edges = set() @@ -124,8 +119,7 @@ async def test_ece_small_graph_min_loss(): assert len(all_edges) == 7 -@pytest.mark.asyncio -async def test_ece_small_graph_max_loss(): +def test_ece_small_graph_max_loss(): """ Same grid graph, but using max_loss sampling. Edges with higher loss should be preferred during community expansion. @@ -147,14 +141,14 @@ async def test_ece_small_graph_max_loss(): with tempfile.TemporaryDirectory() as tmpdir: storage = NetworkXStorage(working_dir=tmpdir, namespace="small_max") for nid, ndata in nodes: - await storage.upsert_node(nid, ndata) + storage.upsert_node(nid, ndata) for src, tgt, edata in edges: - await storage.upsert_edge(src, tgt, edata) + storage.upsert_edge(src, tgt, edata) partitioner = ECEPartitioner() - communities: list[Community] = await partitioner.partition( + communities: list[Community] = list(partitioner.partition( storage, max_units_per_community=4, unit_sampling="max_loss" - ) + )) all_nodes = set() all_edges = set() @@ -166,8 +160,7 @@ async def test_ece_small_graph_max_loss(): assert len(all_edges) == 7 -@pytest.mark.asyncio -async def test_ece_max_tokens_limit(): +def test_ece_max_tokens_limit(): """Ensure max_tokens_per_community is respected.""" # node id -> data node_data = {"A": {"length": 3000}, "B": {"length": 3000}, "C": {"length": 3000}} @@ -177,17 +170,17 @@ async def test_ece_max_tokens_limit(): with tempfile.TemporaryDirectory() as tmpdir: storage = NetworkXStorage(working_dir=tmpdir, namespace="token_limit") for nid, ndata in node_data.items(): - await storage.upsert_node(nid, ndata) + storage.upsert_node(nid, ndata) for src, tgt, edata in edges: - await storage.upsert_edge(src, tgt, edata) + storage.upsert_edge(src, tgt, edata) partitioner = ECEPartitioner() - communities: list[Community] = await partitioner.partition( + communities: list[Community] = list(partitioner.partition( storage, max_units_per_community=10, max_tokens_per_community=5000, # 1 node (3000) + 1 edge (2000) = 5000 unit_sampling="random", - ) + )) # With a 5000-token budget we need at least two communities assert len(communities) >= 2 diff --git a/tests/test_event_profiles.py b/tests/test_event_profiles.py new file mode 100644 index 00000000..fd88539b --- /dev/null +++ b/tests/test_event_profiles.py @@ -0,0 +1,20 @@ +from graphgen.models.generator.support import build_generation_view, validate_support + + +def test_generation_view_hides_event_ids_and_maps_support_back_to_canonical_id(): + event_id = "event:opaque-hash" + item = { + "nodes": [ + (event_id, {"entity_type": "EVENT", "graph_role": "event", "description": "Migration completed：CASE-A7 event fact"}), + ("Operator", {"entity_type": "ACTOR", "description": "performed the action"}), + ], + "edges": [ + (event_id, "Operator", {"description": "participated"}), + ], + } + view = build_generation_view(item) + assert view["nodes"][0][0] == "Migration completed" + assert event_id not in str(view["nodes"]) + mapped = view["resolve_support"]({"cited": ["Migration completed"]}) + assert mapped == {"cited": [event_id]} + assert validate_support(mapped, set(view["canonical_nodes"])) == mapped diff --git a/tests/test_package_profiles.py b/tests/test_package_profiles.py new file mode 100644 index 00000000..da10c7e9 --- /dev/null +++ b/tests/test_package_profiles.py @@ -0,0 +1,30 @@ +from pathlib import Path + +REPO = Path(__file__).resolve().parents[1] + + +def test_event_profile_assets_are_in_source_manifest_and_wheel(tmp_path): + import subprocess + import tarfile + import zipfile + + output = tmp_path / "dist" + subprocess.run(["uv", "build", str(REPO), "--out-dir", str(output)], check=True, capture_output=True, text=True) + wheel = next(output.glob("*.whl")) + sdist = next(output.glob("*.tar.gz")) + expected = { + "pms_event.v1/profile.json", + "pms_event.v1/prompt.zh.txt", + "generic_event.v1/profile.json", + "generic_event.v1/prompt.zh.txt", + "example_event.v1/profile.json", + "example_event.v1/prompt.zh.txt", + "pms_policy.v2/atomic.json", + "pms_policy.v3/multi_hop.json", + } + with zipfile.ZipFile(wheel) as archive: + wheel_files = archive.namelist() + with tarfile.open(sdist) as archive: + sdist_files = archive.getnames() + assert all(any(path.endswith(item) for path in wheel_files) for item in expected) + assert all(any(path.endswith(item) for path in sdist_files) for item in expected) From e11158cd8254f98c827690b60648373149f8a8f5 Mon Sep 17 00:00:00 2001 From: Leon-L20 Date: Fri, 16 Oct 2026 05:39:56 +0000 Subject: [PATCH 5/9] =?UTF-8?q?feat:=20=E5=8F=91=E5=B8=83=20GraphGen=201.1?= =?UTF-8?q?.0=20=E8=BF=90=E8=A1=8C=E6=97=B6=E7=A8=B3=E5=AE=9A=E6=80=A7?= =?UTF-8?q?=E8=83=BD=E5=8A=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增可配置 Ray LLM actor 并发并保持默认串行兼容。 为缺失 logger 上下文的 Ray 调用提供标准 logger fallback；加入并发与日志回归测试，更新版本及发行说明。 Co-Authored-By: Claude Opus 5.5 --- docs/PMS_PATCHES.md | 12 ++++ docs/RELEASE_NOTES-1.0.0.md | 2 +- docs/RELEASE_NOTES-1.1.0.md | 21 +++++++ graphgen/_version.py | 2 +- graphgen/common/init_llm.py | 19 ++++-- graphgen/utils/log.py | 6 +- .../utils/test_init_llm_actor_concurrency.py | 58 +++++++++++++++++++ tests/unit_tests/utils/test_log.py | 12 ++++ 8 files changed, 121 insertions(+), 11 deletions(-) create mode 100644 docs/RELEASE_NOTES-1.1.0.md create mode 100644 tests/unit_tests/utils/test_init_llm_actor_concurrency.py create mode 100644 tests/unit_tests/utils/test_log.py diff --git a/docs/PMS_PATCHES.md b/docs/PMS_PATCHES.md index 54736cad..6682faa8 100644 --- a/docs/PMS_PATCHES.md +++ b/docs/PMS_PATCHES.md @@ -93,3 +93,15 @@ SAG 思想移植（MIT，arXiv:2606.15971；参考源码 `workspace/sag_referenc `tests/test_run_graphgen_defaults.py` 为契约回归（rft 环境按路径加载 纯 stdlib 模块校验）； - 事件 profile 合同和真实生成由 GraphGen venv smoke 验收。 + +## #9 Ray LLM actor concurrency and logger context fallback (1.1.0) + +- `LLMFactory.create_llm` reads `GRAPHGEN_LLM_ACTOR_MAX_CONCURRENCY`; default/`1` preserve Ray's serial async actor behavior, values greater than one pass an explicit bounded `max_concurrency` actor option. +- `get_current_logger` falls back to the standard `GraphGen` logger only when the ContextVar is absent. This does not catch or replace LLM/model errors and does not fabricate generation output. +- Covered by `tests/unit_tests/utils/test_init_llm_actor_concurrency.py` and `tests/unit_tests/utils/test_log.py`. + +## #9 Ray LLM actor concurrency and logger context fallback (1.1.0) + +- `LLMFactory.create_llm` reads `GRAPHGEN_LLM_ACTOR_MAX_CONCURRENCY`; default/`1` preserve Ray's serial async actor behavior, values greater than one pass an explicit bounded `max_concurrency` actor option. +- `get_current_logger` falls back to the standard `GraphGen` logger only when the ContextVar is absent. This does not catch or replace LLM/model errors and does not fabricate generation output. +- Covered by `tests/unit_tests/utils/test_init_llm_actor_concurrency.py` and `tests/unit_tests/utils/test_log.py`. diff --git a/docs/RELEASE_NOTES-1.0.0.md b/docs/RELEASE_NOTES-1.0.0.md index 4297347c..e9dc53e7 100644 --- a/docs/RELEASE_NOTES-1.0.0.md +++ b/docs/RELEASE_NOTES-1.0.0.md @@ -1,6 +1,6 @@ # GraphGen Fork 1.0.0 — initial multi-project SDK release -Tag: `graphgen-fork-v1.0.0` (to be created only after passing release validation). +Tag: `graphgen-fork-v1.0.0`. Distribution version: `1.0.0` (`graphg`). Upstream base: `InternScience/GraphGen` commit `3a3eb097318d34b07f6f31cb60de5b20712d65ce`. diff --git a/docs/RELEASE_NOTES-1.1.0.md b/docs/RELEASE_NOTES-1.1.0.md new file mode 100644 index 00000000..e9653d20 --- /dev/null +++ b/docs/RELEASE_NOTES-1.1.0.md @@ -0,0 +1,21 @@ +# GraphGen Fork 1.1.0 — production runtime fixes + +Tag: `graphgen-fork-v1.1.0`. + +Distribution version: `1.1.0` (`graphg`). Base: `graphgen-fork-v1.0.0` commit `c5ef453aa661776b757902a990c529fdab2bfc9a`. + +## Changes + +- Added bounded, opt-in Ray LLM actor concurrency through `GRAPHGEN_LLM_ACTOR_MAX_CONCURRENCY`. The default remains Ray's existing concurrency of 1; values greater than 1 are passed as the actor's `max_concurrency` option. +- Logger calls from Ray contexts without a propagated logger context now use the `GraphGen` standard logger instead of raising during logging. This fallback changes logging only; model errors and generated output semantics are unchanged. +- PMS prompt policy remains an independently versioned consumer policy. This release does not overwrite the immutable v1.0.0 prompt assets. + +## Validation + +- New actor concurrency and logger context tests: 4 passed. +- Event profile and package tests: 2 passed when combined with the new unit tests. +- Wheel and sdist built; each contains actor and logger modules plus all four generation profile JSON files. +- Clean Python 3.12 venv installed the wheel. `graphg` reported 1.1.0; both built-in PMS policy profiles loaded; the `pms_project_anchor` output gate worked. +- Wheel and sdist SHA-256 values are recorded alongside the exact release commit in the release artifact manifest. + +PMS full production integration is validated separately in the PMS repository, which pins this fork commit explicitly and owns PMS policy files. diff --git a/graphgen/_version.py b/graphgen/_version.py index 43fe2c19..1c9d3d9f 100644 --- a/graphgen/_version.py +++ b/graphgen/_version.py @@ -1,6 +1,6 @@ from typing import Tuple -__version__ = "1.0.0" +__version__ = "1.1.0" short_version = __version__ diff --git a/graphgen/common/init_llm.py b/graphgen/common/init_llm.py index 56bffedf..c690ebea 100644 --- a/graphgen/common/init_llm.py +++ b/graphgen/common/init_llm.py @@ -129,19 +129,26 @@ def create_llm( ) actor_name = f"Actor_LLM_{model_type}" + num_gpus = float(config.pop("num_gpus", 0)) + # Ray async actor 默认 max_concurrency=1：所有 worker 的 LLM 请求在 + # 单 actor 内串行，是 GraphGen 出题吞吐的瓶颈。PMS fork 允许经环境 + # 变量调高并发（async 方法并发安全），上游默认行为不变（未设置=1）。 + max_concurrency = int(os.environ.get("GRAPHGEN_LLM_ACTOR_MAX_CONCURRENCY", "1")) + actor_options: Dict[str, Any] = { + "name": actor_name, + "num_gpus": num_gpus, + "get_if_exists": True, + } + if max_concurrency > 1: + actor_options["max_concurrency"] = max_concurrency try: actor_handle = ray.get_actor(actor_name) print(f"Using existing Ray actor: {actor_name}") except ValueError: print(f"Creating Ray actor for LLM {model_type} with backend {backend}.") - num_gpus = float(config.pop("num_gpus", 0)) actor_handle = ( ray.remote(LLMServiceActor) - .options( - name=actor_name, - num_gpus=num_gpus, - get_if_exists=True, - ) + .options(**actor_options) .remote(backend, config) ) diff --git a/graphgen/utils/log.py b/graphgen/utils/log.py index e29e994e..abaa587b 100644 --- a/graphgen/utils/log.py +++ b/graphgen/utils/log.py @@ -62,9 +62,9 @@ def set_logger( def get_current_logger() -> logging.Logger: - current_logger = CURRENT_LOGGER_VAR.get() - if not current_logger: - raise RuntimeError("No logger is set in the current context.") + current_logger = CURRENT_LOGGER_VAR.get(None) + if current_logger is None: + return logging.getLogger("GraphGen") return current_logger diff --git a/tests/unit_tests/utils/test_init_llm_actor_concurrency.py b/tests/unit_tests/utils/test_init_llm_actor_concurrency.py new file mode 100644 index 00000000..a8e3920d --- /dev/null +++ b/tests/unit_tests/utils/test_init_llm_actor_concurrency.py @@ -0,0 +1,58 @@ +"""Ray synthesizer actor concurrency remains bounded and opt-in.""" +from __future__ import annotations + +import sys + +import pytest + +from graphgen.common.init_llm import LLMFactory + + +@pytest.mark.parametrize( + ("configured", "expected"), + [(None, None), ("1", None), ("4", 4)], +) +def test_actor_max_concurrency_is_opt_in( + monkeypatch: pytest.MonkeyPatch, configured: str | None, expected: int | None +) -> None: + options_seen: dict = {} + + class FakeRemoteMethod: + def remote(self): + return "ready" + + class FakeHandle: + ready = FakeRemoteMethod() + + class FakeRemote: + def options(self, **options): + options_seen.update(options) + return self + + def remote(self, backend, config): + return FakeHandle() + + class FakeRay: + @staticmethod + def get_actor(_name): + raise ValueError("missing") + + @staticmethod + def remote(_actor): + return FakeRemote() + + @staticmethod + def get(value): + return value + + monkeypatch.setitem(sys.modules, "ray", FakeRay) + if configured is None: + monkeypatch.delenv("GRAPHGEN_LLM_ACTOR_MAX_CONCURRENCY", raising=False) + else: + monkeypatch.setenv("GRAPHGEN_LLM_ACTOR_MAX_CONCURRENCY", configured) + + result = LLMFactory.create_llm("synthesizer", "fake", {"num_gpus": 0}) + + assert result is not None + assert options_seen["get_if_exists"] is True + assert options_seen.get("max_concurrency") == expected diff --git a/tests/unit_tests/utils/test_log.py b/tests/unit_tests/utils/test_log.py new file mode 100644 index 00000000..c17e0d13 --- /dev/null +++ b/tests/unit_tests/utils/test_log.py @@ -0,0 +1,12 @@ +import logging +from contextvars import Context + +from graphgen.utils.log import logger + + +def test_context_aware_logger_handles_ray_context_without_operator_context(caplog): + """LLM Ray actors may log without inheriting an operator's ContextVar.""" + with caplog.at_level(logging.WARNING): + Context().run(logger.warning, "empty LLM response from actor") + + assert "empty LLM response from actor" in caplog.text From 0692702a91d5ea35b468543965998d7877f5700c Mon Sep 17 00:00:00 2001 From: Leon-L20 Date: Fri, 16 Oct 2026 07:34:29 +0000 Subject: [PATCH 6/9] =?UTF-8?q?fix:=20=E4=BA=8B=E4=BB=B6=E6=8A=BD=E5=8F=96?= =?UTF-8?q?=E6=8F=90=E7=A4=BA=E8=AF=8D=E7=A6=81=E6=AD=A2=E7=85=A7=E6=8A=84?= =?UTF-8?q?=E7=A4=BA=E4=BE=8B=E9=A1=B9=E7=9B=AE=E7=BC=96=E5=8F=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 内置 pms_event.v1 提示词要求锚点必须原样取自片段，并加入防照抄回归测试；与生成策略 v2_2 的加固一致。版本号升至 1.1.1。 Co-Authored-By: Claude Opus 5.5 --- docs/PMS_PATCHES.md | 4 ++++ docs/RELEASE_NOTES-1.1.1.md | 16 ++++++++++++++++ graphgen/_version.py | 2 +- .../event_profiles/pms_event.v1/prompt.zh.txt | 2 +- tests/test_event_profiles.py | 10 ++++++++++ 5 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 docs/RELEASE_NOTES-1.1.1.md diff --git a/docs/PMS_PATCHES.md b/docs/PMS_PATCHES.md index 6682faa8..b136e639 100644 --- a/docs/PMS_PATCHES.md +++ b/docs/PMS_PATCHES.md @@ -105,3 +105,7 @@ SAG 思想移植（MIT，arXiv:2606.15971；参考源码 `workspace/sag_referenc - `LLMFactory.create_llm` reads `GRAPHGEN_LLM_ACTOR_MAX_CONCURRENCY`; default/`1` preserve Ray's serial async actor behavior, values greater than one pass an explicit bounded `max_concurrency` actor option. - `get_current_logger` falls back to the standard `GraphGen` logger only when the ContextVar is absent. This does not catch or replace LLM/model errors and does not fabricate generation output. - Covered by `tests/unit_tests/utils/test_init_llm_actor_concurrency.py` and `tests/unit_tests/utils/test_log.py`. + +## #10 Event extraction prompt: no example-id copying (1.1.1) + +- `templates/event_profiles/pms_event.v1/prompt.zh.txt` requires the anchor to be taken verbatim from the fragment and forbids copying the illustrative id, matching the generation-policy hardening in PMS `pms_policy.v2_2`. diff --git a/docs/RELEASE_NOTES-1.1.1.md b/docs/RELEASE_NOTES-1.1.1.md new file mode 100644 index 00000000..b2665fb4 --- /dev/null +++ b/docs/RELEASE_NOTES-1.1.1.md @@ -0,0 +1,16 @@ +# GraphGen Fork 1.1.1 — event extraction prompt fix + +Tag: `graphgen-fork-v1.1.1`. + +Distribution version: `1.1.1` (`graphg`). Base: `graphgen-fork-v1.1.0` commit `e11158cd8254f98c827690b60648373149f8a8f5`. + +## Changes + +- Built-in `pms_event.v1` extraction prompt now forbids copying the example project id: the anchor must be taken verbatim from the graph fragment. This mirrors the generation-policy fix that removed the previously observed defect where example ids leaked into production output. +- No API, schema, or default-route change. LightRAG + ECE remains the default. + +## Validation + +- GraphGen unit, event profile, and package tests pass in the pinned environment. +- Wheel and sdist are built and inspected; the fixed prompt ships inside both. +- Consumer verification (install, profile load, end-to-end extraction and generation) is recorded in the PMS repository that pins this commit. diff --git a/graphgen/_version.py b/graphgen/_version.py index 1c9d3d9f..6f453082 100644 --- a/graphgen/_version.py +++ b/graphgen/_version.py @@ -1,6 +1,6 @@ from typing import Tuple -__version__ = "1.1.0" +__version__ = "1.1.1" short_version = __version__ diff --git a/graphgen/templates/event_profiles/pms_event.v1/prompt.zh.txt b/graphgen/templates/event_profiles/pms_event.v1/prompt.zh.txt index 7ae0749f..de0d7ad9 100644 --- a/graphgen/templates/event_profiles/pms_event.v1/prompt.zh.txt +++ b/graphgen/templates/event_profiles/pms_event.v1/prompt.zh.txt @@ -12,7 +12,7 @@ - "计划、预计、建议、可能、正在、已经完成"必须严格区分。 - 不得把外部常识、文件名或模型推测写成原文事实。 - 每个事项必须锚定在一个明确项目上：content 中必须出现该项目的完整项目编号 - （形如 P-24084947、I-26015101，字母+横线+至少 6 位数字）。无法锚定到完整项目 + （形如 P-00000000，编号必须原样取自图谱片段中的项目节点，严禁照抄示例编号，字母+横线+至少 6 位数字）。无法锚定到完整项目 编号的事项不得输出。 - 项目名称在同系统可能不唯一：提及时必须连同完整项目编号一起出现。 - 字段锁定：只允许陈述项目编号、项目名称、项目阶段、最终用户、销售工程师、 diff --git a/tests/test_event_profiles.py b/tests/test_event_profiles.py index fd88539b..c519a457 100644 --- a/tests/test_event_profiles.py +++ b/tests/test_event_profiles.py @@ -18,3 +18,13 @@ def test_generation_view_hides_event_ids_and_maps_support_back_to_canonical_id() mapped = view["resolve_support"]({"cited": ["Migration completed"]}) assert mapped == {"cited": [event_id]} assert validate_support(mapped, set(view["canonical_nodes"])) == mapped + + +def test_pms_event_prompt_forbids_copying_the_example_project_id(): + """Regression: example ids must never be presented as copyable anchors.""" + from graphgen.templates.event_profile_loader import load_event_profile + + prompt = load_event_profile("pms_event.v1").prompt + assert "严禁照抄" in prompt + assert "P-00000000" in prompt + assert "P-24084947" not in prompt From 49ec1ac8dfe9c15204f979c49887f5614a5adf6d Mon Sep 17 00:00:00 2001 From: Leon-L20 Date: Fri, 16 Oct 2026 09:27:45 +0000 Subject: [PATCH 7/9] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E4=BA=8B=E4=BB=B6?= =?UTF-8?q?=E7=AE=A1=E7=BA=BF=E5=BC=95=E7=94=A8=E4=B8=8E=E9=87=8D=E8=AF=95?= =?UTF-8?q?=E7=BC=BA=E9=99=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复文件型事件 profile 传递、双向 support 边引用、无锚事件分区、JSON prefill 和逐请求限流；保留事件成员标记与角色属性。升级 graphg 到 1.1.2，补充回归测试和发行记录。 Co-Authored-By: Claude Opus 5.5 --- docs/PMS_PATCHES.md | 14 +- docs/RELEASE_MANIFEST-1.1.2.md | 41 +++ docs/RELEASE_NOTES-1.1.2.md | 30 +++ graphgen/_version.py | 2 +- .../extractor/schema_guided_extractor.py | 2 +- graphgen/models/generator/support.py | 33 ++- .../kg_builder/event_entity_kg_builder.py | 6 + graphgen/models/llm/api/openai_client.py | 8 +- .../partitioner/event_join_partitioner.py | 2 +- .../operators/build_kg/build_kg_service.py | 2 +- .../operators/generate/generate_service.py | 6 +- .../test_event_profile_loader_regressions.py | 247 ++++++++++++++++++ tests/test_support_edge_citations.py | 49 ++++ 13 files changed, 416 insertions(+), 26 deletions(-) create mode 100644 docs/RELEASE_MANIFEST-1.1.2.md create mode 100644 docs/RELEASE_NOTES-1.1.2.md create mode 100644 tests/test_event_profile_loader_regressions.py create mode 100644 tests/test_support_edge_citations.py diff --git a/docs/PMS_PATCHES.md b/docs/PMS_PATCHES.md index b136e639..951c9928 100644 --- a/docs/PMS_PATCHES.md +++ b/docs/PMS_PATCHES.md @@ -64,10 +64,10 @@ SAG 思想移植（MIT，arXiv:2606.15971；参考源码 `workspace/sag_referenc - `models/kg_builder/event_contract.py`：纯 stdlib 响应合同（禁合同外字段、is_valid 自报、 无锚 fail-closed 丢弃并计数）+ 数字接地校验（SAG grounding 移植）+ 确定性事件 ID。 - `models/kg_builder/event_entity_kg_builder.py`：事件→星型编码（EVENT 节点 + 成员边， - 边属性含角色）。实体描述/普通边继续复用官方合并；**EVENT 节点与事件成员边用确定性 - SEP 合并、不走 LLM 摘要**（SAG 事件是事实文本，不能让 merge summarizer 把 event_id - 扩写成“项目身份/分工总结”污染出题上下文）。事件描述限长 4,000 字符，且原文链以 - `source_id` SEP 保持闭合。 + 边属性含角色）。源实现对事件节点、参与者节点及成员边执行确定性合并；仅普通未标记边 + 委托官方 LightRAG merge。**事件节点与事件成员边不走 LLM 摘要**（SAG 事件是事实文本， + 不能让 merge summarizer 把 event_id 扩写成“项目身份/分工总结”污染出题上下文）。事件 + 描述限长 4,000 字符，且原文链以 `source_id` SEP 保持闭合。 - `operators/build_kg/`：`kg_method: event_entity` 分发（默认 light_rag 与官方逐字节一致）。 - `models/partitioner/event_join_partitioner.py`：共享实体 join + 同锚硬约束的事件簇分区 （SAG join 语义离线物化），产出官方 `Community`，metadata 携带 event_ids。 @@ -100,12 +100,6 @@ SAG 思想移植（MIT，arXiv:2606.15971；参考源码 `workspace/sag_referenc - `get_current_logger` falls back to the standard `GraphGen` logger only when the ContextVar is absent. This does not catch or replace LLM/model errors and does not fabricate generation output. - Covered by `tests/unit_tests/utils/test_init_llm_actor_concurrency.py` and `tests/unit_tests/utils/test_log.py`. -## #9 Ray LLM actor concurrency and logger context fallback (1.1.0) - -- `LLMFactory.create_llm` reads `GRAPHGEN_LLM_ACTOR_MAX_CONCURRENCY`; default/`1` preserve Ray's serial async actor behavior, values greater than one pass an explicit bounded `max_concurrency` actor option. -- `get_current_logger` falls back to the standard `GraphGen` logger only when the ContextVar is absent. This does not catch or replace LLM/model errors and does not fabricate generation output. -- Covered by `tests/unit_tests/utils/test_init_llm_actor_concurrency.py` and `tests/unit_tests/utils/test_log.py`. - ## #10 Event extraction prompt: no example-id copying (1.1.1) - `templates/event_profiles/pms_event.v1/prompt.zh.txt` requires the anchor to be taken verbatim from the fragment and forbids copying the illustrative id, matching the generation-policy hardening in PMS `pms_policy.v2_2`. diff --git a/docs/RELEASE_MANIFEST-1.1.2.md b/docs/RELEASE_MANIFEST-1.1.2.md new file mode 100644 index 00000000..e31233bc --- /dev/null +++ b/docs/RELEASE_MANIFEST-1.1.2.md @@ -0,0 +1,41 @@ +# GraphGen 1.1.2 release manifest + +Status: validation evidence for the release candidate; commit SHA is recorded after final release commit. + +## Source identity + +- Distribution: `graphg` 1.1.2 +- Candidate base/tag: `graphgen-fork-v1.1.1` / `0692702a91d5ea35b468543965998d7877f5700c` +- Upstream base: `origin/main` `3a3eb097318d34b07f6f31cb60de5b20712d65ce` +- Intended publication remote: `fork` (`https://github.com/Leon-Algo/GraphGen.git`); upstream `origin` is not to be modified. +- PMS consumer policy remains PMS-owned; no PMS prompt files are included in the GraphGen diff. + +## Test and smoke evidence + +Run in the GraphGen defect worktree with Python 3.12 dependencies supplied by `workspace/graphgen_env/.venv`; `PYTHONPATH` must put the defect worktree before the installed wheel when testing source changes. + +- `PYTHONPATH=: UV_OFFLINE=1 NLTK_DISABLE_IMPORT_SECURITY=1 -m pytest -c /dev/null -q tests/unit_tests tests/test_event_profile_loader_regressions.py tests/test_support_edge_citations.py tests/test_event_profiles.py` — **15 passed**. +- `PYTHONPATH=: UV_OFFLINE=1 NLTK_DISABLE_IMPORT_SECURITY=1 -m pytest -c /dev/null -q tests/test_package_profiles.py` — **1 passed**. +- `PYTHONPATH=: NLTK_DISABLE_IMPORT_SECURITY=1 -m pytest -c /dev/null -q /tests/test_pms_graphgen_policy_generation.py /tests/test_run_graphgen_defaults.py /tests/test_repair_project_query_graphgen_chunk_cache.py` — **28 passed, 1 skipped**. +- `PYTHONPATH=: NLTK_DISABLE_IMPORT_SECURITY=1 -m pytest -c /dev/null -q tests/integration_tests/models/partitioner/test_ece_partitioner.py` — **6 passed**. +- Non-PMS deterministic integration smoke via `EventEntityKGBuilder` extraction+merge → `EventJoinPartitioner`: **3 stub LLM calls, 3 event nodes, 3 membership edges, 3 communities; role persisted on every edge; anchors CASE-A/CASE-B/CASE-C; no network/model calls**. Executed by the GraphGen test agent; command: `PYTHONPATH= /.venv/bin/python /tmp/graphgen_event_join_smoke.py`. +- `git diff --check` — passed. + +## Build and clean-install evidence + +Built in a disposable output directory with `uv build --offline --out-dir /tmp/graphgen-release-1.1.2`: + +- `graphg-1.1.2-py3-none-any.whl` — SHA-256 `5272d7c6a03eaa3d8add95b2674946a0f20895a9a420e3f547ee04a54477f792` +- `graphg-1.1.2.tar.gz` — SHA-256 `234d0ec6302a92077fda5e9edfd35b542784de4bf50841e6dab3ccef5474cab3` +- Archive inspection confirmed `profile.json` and `prompt.zh.txt` for `example_event.v1`, `generic_event.v1`, and `pms_event.v1` in both artifacts. +- Installed the wheel with `uv pip install --offline --python /tmp/graphgen-clean312/bin/python --no-deps --force-reinstall ` into a fresh Python 3.12 venv. Confirmed `graphg==1.1.2`, `graphgen.__file__` under that venv's `site-packages`, and loaded all three built-in event profiles. Prompt hashes: + - `example_event.v1`: `8a27caf4453f9734a499a63c97ee68c5cb0d73297409df86a6d82461af31c828` + - `generic_event.v1`: `801aa5cd8967d608dbdcf5d9057fe7c91e9154862516be85944c335207750136` + - `pms_event.v1`: `7ab5656e8b80a90310a1a7645ae0b11b752127a5628d1b3de2994d9929552b25` + +## Publication record + +- Final commit SHA: pending +- Tag creation: pending; must verify local and `fork` remote tag absence first. +- Fork branch/tag push: pending maintainer action after final diff review. +- PMS dependency pin update and locked sync: pending 1.1.2 commit SHA after publication. diff --git a/docs/RELEASE_NOTES-1.1.2.md b/docs/RELEASE_NOTES-1.1.2.md new file mode 100644 index 00000000..6b26b19f --- /dev/null +++ b/docs/RELEASE_NOTES-1.1.2.md @@ -0,0 +1,30 @@ +# GraphGen Fork 1.1.2 — event pipeline correctness fixes + +Tag: `graphgen-fork-v1.1.2`. + +Distribution version: `1.1.2` (`graphg`). Base: `graphgen-fork-v1.1.1` commit `0692702a91d5ea35b468543965998d7877f5700c`. + +## Changes + +- Preserve explicit filesystem event-profile references through build-service validation so custom project profiles remain loadable by the event builder. +- Resolve support citations using the rendered edge separator and either displayed endpoint order; validate citations only against canonical edges in the partition, while retaining node-only validation for existing callers. +- Skip unanchored events when same-anchor event partitioning is required instead of aborting the partition run. +- Use a JSON-compatible `{` assistant prefill for the opt-in empty schema-extraction retry, and reapply RPM/TPM limits before every actual API attempt. +- Mark event membership edges in direct extraction results and persist their participant role through deterministic edge merging. +- Correct the patch ledger's merge-path description and remove its duplicate concurrency/logger entry. + +No default route changes: LightRAG extraction and ECE partitioning remain the default. PMS profiles remain PMS-owned and independently versioned. + +## Validation + +- GraphGen event, support, profile and package regression tests: see release manifest for commands and results. +- PMS strategy, runner and cache-repair compatibility tests: see release manifest. +- Non-PMS event extraction → star graph → event_join smoke: see release manifest. +- Python 3.12 wheel installation and built-in profile loading: see release manifest. +- Wheel and sdist SHA-256, final commit, profile asset hashes and upstream base are recorded in `RELEASE_MANIFEST-1.1.2.md`. + +## Compatibility + +- No migration is required for existing configs or graph caches. +- This release only fixes event-profile references, citations, retry accounting, anchor handling and event-member metadata. +- The upstream repository and PR #192 are unchanged. diff --git a/graphgen/_version.py b/graphgen/_version.py index 6f453082..bfa4e616 100644 --- a/graphgen/_version.py +++ b/graphgen/_version.py @@ -1,6 +1,6 @@ from typing import Tuple -__version__ = "1.1.1" +__version__ = "1.1.2" short_version = __version__ diff --git a/graphgen/models/extractor/schema_guided_extractor.py b/graphgen/models/extractor/schema_guided_extractor.py index f594350f..fd404dd9 100644 --- a/graphgen/models/extractor/schema_guided_extractor.py +++ b/graphgen/models/extractor/schema_guided_extractor.py @@ -72,7 +72,7 @@ async def extract(self, chunk: Chunk) -> dict: # that seeds the opening of the expected output; non-empty responses # are never retried or modified. if not response.strip() and os.environ.get("GRAPHGEN_EMPTY_EXTRACTION_PREFILL", "0") == "1": - prefill = '("entity"<|>' + prefill = "{" logger.warning("Empty extraction response; retrying with assistant prefill") response = await self.llm_client.generate_answer( prompt, assistant_prefill=prefill diff --git a/graphgen/models/generator/support.py b/graphgen/models/generator/support.py index d66dd633..4007cfcf 100644 --- a/graphgen/models/generator/support.py +++ b/graphgen/models/generator/support.py @@ -63,6 +63,7 @@ def build_generation_view(item: dict[str, Any]) -> dict[str, Any]: for node_id, data in original_nodes ] edges = [] + canonical_edges = set() support_aliases = {name: node_id for node_id, name in display_by_id.items()} for edge in item.get("edges", []): if not isinstance(edge, (list, tuple)) or len(edge) != 3: @@ -71,10 +72,17 @@ def build_generation_view(item: dict[str, Any]) -> dict[str, Any]: display_source = display_by_id.get(str(source), source) display_target = display_by_id.get(str(target), target) edges.append((display_source, display_target, data)) - if str(source) in display_by_id: - support_aliases[f"{display_source} - {display_target}"] = f"{source} - {target}" - elif str(target) in display_by_id: - support_aliases[f"{display_source} - {display_target}"] = f"{source} - {target}" + canonical_edge = f"{source} - {target}" + canonical_edges.add(canonical_edge) + if str(source) in display_by_id or str(target) in display_by_id: + support_aliases[f"{display_source} -- {display_target}"] = canonical_edge + support_aliases[f"{display_source} - {display_target}"] = canonical_edge + support_aliases[f"{display_target} -- {display_source}"] = ( + f"{target} - {source}" + ) + support_aliases[f"{display_target} - {display_source}"] = ( + f"{target} - {source}" + ) def resolve_support(support: Optional[dict[str, Any]]) -> Optional[dict[str, Any]]: if not support: @@ -91,15 +99,26 @@ def resolve_support(support: Optional[dict[str, Any]]) -> Optional[dict[str, Any "nodes": nodes, "edges": edges, "canonical_nodes": [str(node_id) for node_id, _ in original_nodes], + "canonical_edges": canonical_edges, "resolve_support": resolve_support, } -def validate_support(support: Optional[dict[str, Any]], node_names: set[str]) -> Optional[dict[str, Any]]: - """Keep support only when all citations point to existing partition nodes.""" +def validate_support( + support: Optional[dict[str, Any]], + node_names: set[str], + edge_names: Optional[set[str]] = None, +) -> Optional[dict[str, Any]]: + """Keep support only when citations reference known partition nodes or edges.""" if not support: return None cited = support.get("cited") or [] - if cited and set(cited) <= node_names: + if not cited: + return None + if all( + citation in node_names + or (edge_names is not None and citation in edge_names) + for citation in cited + ): return support return None diff --git a/graphgen/models/kg_builder/event_entity_kg_builder.py b/graphgen/models/kg_builder/event_entity_kg_builder.py index f7dfee7f..f180ffc6 100644 --- a/graphgen/models/kg_builder/event_entity_kg_builder.py +++ b/graphgen/models/kg_builder/event_entity_kg_builder.py @@ -115,6 +115,7 @@ async def extract( "description": f"事件参与者：{entity['name']}（{entity['type']}）—— {entity['description']}", "source_id": chunk.id, "role": entity["type"], + "event_member": True, } ) return dict(nodes), dict(edges) @@ -197,6 +198,9 @@ async def merge_edges(self, edges_data: tuple, kg_instance: BaseGraphStorage) -> } | { sid for sid in str(existing_edge.get("source_id", "")).split("") if sid } + role = sorted( + {str(dp["role"]) for dp in edge_list if dp.get("role")} + ) edge_data = { "src_id": event_id, "tgt_id": participant_id, @@ -204,6 +208,8 @@ async def merge_edges(self, edges_data: tuple, kg_instance: BaseGraphStorage) -> "source_id": "".join(sorted(source_ids)), "length": self._merger.tokenizer.count_tokens(merged), } + if role: + edge_data["role"] = "".join(role) kg_instance.upsert_edge(event_id, participant_id, edge_data=edge_data) return edge_data return await self._merger.merge_edges(edges_data, kg_instance=kg_instance) diff --git a/graphgen/models/llm/api/openai_client.py b/graphgen/models/llm/api/openai_client.py index 916424af..e9632b06 100644 --- a/graphgen/models/llm/api/openai_client.py +++ b/graphgen/models/llm/api/openai_client.py @@ -179,10 +179,6 @@ async def generate_answer( prompt_tokens += len(self.tokenizer.encode(message["content"])) estimated_tokens = prompt_tokens + kwargs["max_tokens"] - if self.request_limit: - await self.rpm.wait(silent=True) - await self.tpm.wait(estimated_tokens, silent=True) - # PMS fork (docs/PMS_PATCHES.md #4): bounded retry on empty responses. # Thinking backends occasionally return only content that is # stripped to nothing downstream; retry instead of propagating an empty @@ -197,6 +193,10 @@ async def generate_answer( content = "" for attempt in range(empty_retries + 1): + if self.request_limit: + await self.rpm.wait(silent=True) + await self.tpm.wait(estimated_tokens, silent=True) + completion = await self.client.chat.completions.create( # pylint: disable=E1125 model=self.model, **kwargs ) diff --git a/graphgen/models/partitioner/event_join_partitioner.py b/graphgen/models/partitioner/event_join_partitioner.py index 327e447d..14d9108a 100644 --- a/graphgen/models/partitioner/event_join_partitioner.py +++ b/graphgen/models/partitioner/event_join_partitioner.py @@ -63,7 +63,7 @@ def _grow(seed: Tuple[str, dict]) -> Optional[Community]: token_sum = 0 anchor = _anchor(seed[0]) if require_same_anchor and not anchor: - raise ValueError(f"event_join_anchor_missing:{seed[0]}") + return None queue = deque([seed[0]]) while queue: diff --git a/graphgen/operators/build_kg/build_kg_service.py b/graphgen/operators/build_kg/build_kg_service.py index c86c8657..0d96cb8a 100644 --- a/graphgen/operators/build_kg/build_kg_service.py +++ b/graphgen/operators/build_kg/build_kg_service.py @@ -34,7 +34,7 @@ def __init__( if self.kg_method == "event_entity": from graphgen.templates.event_profile_loader import load_event_profile - self.event_profile = load_event_profile(self.event_profile).profile_id + load_event_profile(self.event_profile) def process(self, batch: list) -> Tuple[list, dict]: """ diff --git a/graphgen/operators/generate/generate_service.py b/graphgen/operators/generate/generate_service.py index 5f35f2ab..6d21bd1e 100644 --- a/graphgen/operators/generate/generate_service.py +++ b/graphgen/operators/generate/generate_service.py @@ -146,7 +146,11 @@ def process(self, batch: list) -> Tuple[list, dict]: for qa_pair in qa_pairs: support = qa_pair.pop("support", None) if isinstance(qa_pair, dict) else None support = view["resolve_support"](support) - support = validate_support(support, set(view["canonical_nodes"])) + support = validate_support( + support, + set(view["canonical_nodes"]), + set(view["canonical_edges"]), + ) if support: qa_pair["support"] = support res = self.generator.format_generation_results( diff --git a/tests/test_event_profile_loader_regressions.py b/tests/test_event_profile_loader_regressions.py new file mode 100644 index 00000000..10f54ab1 --- /dev/null +++ b/tests/test_event_profile_loader_regressions.py @@ -0,0 +1,247 @@ +from __future__ import annotations + +import json + +import pytest + +from graphgen.templates.event_profile_loader import load_event_profile + + +def _copy_profile(root, profile_id: str = "custom_event.v1"): + source = load_event_profile("example_event.v1") + manifest = { + "schema_version": "graphgen_event_profile.v1", + "profile_id": profile_id, + "prompt_file": "prompt.zh.txt", + "entity_types": list(source.entity_types), + "anchor": {"required": False, "pattern": None, "entity_type": None}, + "grounding": {"number_grounding": False}, + "graph": {"event_entity_type": "EVENT", "legacy_event_id": False}, + } + root.mkdir(parents=True) + (root / "profile.json").write_text(json.dumps(manifest), encoding="utf-8") + (root / "prompt.zh.txt").write_text("{input_text}", encoding="utf-8") + return root / "profile.json" + + +def test_event_member_role_survives_deterministic_edge_merge(): + import asyncio + + from graphgen.models.kg_builder.event_entity_kg_builder import EventEntityKGBuilder + + class Merger: + tokenizer = type("Tokenizer", (), {"count_tokens": lambda _self, text: len(text)})() + + async def merge_edges(self, *_args, **_kwargs): + raise AssertionError("event membership edge must not use LightRAG summary") + + class Storage: + def __init__(self): + self.edge_data = None + + def get_edge(self, *_args): + return None + + def upsert_edge(self, _src, _tgt, edge_data): + self.edge_data = edge_data + + builder = EventEntityKGBuilder.__new__(EventEntityKGBuilder) + builder.profile = load_event_profile("example_event.v1") + builder._merger = Merger() + storage = Storage() + edge = { + "src_id": "event:case-1", + "tgt_id": "Operator", + "description": "participated", + "source_id": "chunk-1", + "role": "ACTOR", + "event_member": True, + } + + merged = asyncio.run( + builder.merge_edges((("event:case-1", "Operator"), [edge]), storage) + ) + + assert merged["role"] == "ACTOR" + assert storage.edge_data["role"] == "ACTOR" +def test_filesystem_event_profile_reference_can_be_reloaded(tmp_path): + """The service-normalized reference preserves the absolute manifest path.""" + manifest = _copy_profile(tmp_path / "profiles" / "custom_event.v1") + reference = str(manifest.resolve()) + + assert reference == str(manifest.resolve()) + assert load_event_profile(reference).prompt == "{input_text}" + assert load_event_profile(reference).prompt == load_event_profile(reference).prompt + + +def test_build_kg_service_preserves_filesystem_event_profile_path(tmp_path, monkeypatch): + import importlib.util + import sys + import types + from pathlib import Path + + from graphgen.templates.event_profile_loader import load_event_profile + + service_module = Path(__file__).parents[1] / "graphgen/operators/build_kg/build_kg_service.py" + package_name = "graphgen.operators.build_kg.build_kg_service_regression" + fake_bases = types.ModuleType("graphgen.bases") + fake_bases.BaseGraphStorage = type("BaseGraphStorage", (), {}) + fake_bases.BaseLLMWrapper = type("BaseLLMWrapper", (), {}) + fake_bases.BaseOperator = type("BaseOperator", (), {"__init__": lambda self, **_kwargs: None}) + fake_datatypes = types.ModuleType("graphgen.bases.datatypes") + fake_datatypes.Chunk = type("Chunk", (), {}) + monkeypatch.setitem(sys.modules, "graphgen.bases", fake_bases) + monkeypatch.setitem(sys.modules, "graphgen.bases.datatypes", fake_datatypes) + monkeypatch.setitem(sys.modules, "graphgen.common.init_llm", types.SimpleNamespace(init_llm=lambda *_args: object())) + monkeypatch.setitem(sys.modules, "graphgen.common.init_storage", types.SimpleNamespace(init_storage=lambda **_kwargs: object())) + monkeypatch.setitem(sys.modules, "graphgen.utils", types.SimpleNamespace(logger=types.SimpleNamespace(info=lambda *_a: None))) + captured_event_profiles = [] + + def capture_event_profile(**kwargs): + captured_event_profiles.append(kwargs["event_profile"]) + return [], [] + + monkeypatch.setitem( + sys.modules, + "graphgen.operators.build_kg.build_event_kg", + types.SimpleNamespace(build_event_entity_kg=capture_event_profile), + ) + monkeypatch.setitem(sys.modules, "graphgen.operators.build_kg.build_mm_kg", types.SimpleNamespace(build_mm_kg=lambda **_kwargs: ([], []))) + monkeypatch.setitem(sys.modules, "graphgen.operators.build_kg.build_text_kg", types.SimpleNamespace(build_text_kg=lambda **_kwargs: ([], []))) + spec = importlib.util.spec_from_file_location(package_name, service_module) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + manifest = _copy_profile(tmp_path / "profiles" / "custom_event.v1") + reference = str(manifest.resolve()) + service = module.BuildKGService.__new__(module.BuildKGService) + module.BuildKGService.__init__(service, kg_method="event_entity", event_profile=reference) + assert service.event_profile == reference + assert load_event_profile(service.event_profile).prompt == "{input_text}" + service.llm_client = object() + service.graph_storage = types.SimpleNamespace(index_done_callback=lambda: None) + fake_datatypes.Chunk.from_dict = lambda _trace_id, doc: types.SimpleNamespace( + type=doc["type"], content=doc["content"] + ) + service.process( + [{"_trace_id": "chunk-1", "type": "text", "content": "project update"}] + ) + assert captured_event_profiles == [reference] + + builtin = module.BuildKGService.__new__(module.BuildKGService) + module.BuildKGService.__init__(builtin, kg_method="event_entity", event_profile="example_event.v1") + assert builtin.event_profile == "example_event.v1" + invalid = module.BuildKGService.__new__(module.BuildKGService) + with pytest.raises(ValueError, match="event_profile_not_found"): + module.BuildKGService.__init__(invalid, kg_method="event_entity", event_profile="/missing/profile.json") + + +def test_event_support_edge_alias_matches_rendered_separator(): + from graphgen.models.generator.support import build_generation_view, validate_support + + event_id = "event:opaque-hash" + item = { + "nodes": [ + (event_id, {"entity_type": "EVENT", "graph_role": "event", "description": "Migration completed：fact"}), + ("Operator", {"entity_type": "ACTOR", "description": "responsible"}), + ], + "edges": [(event_id, "Operator", {"description": "participated"})], + } + view = build_generation_view(item) + citation = f'{view["nodes"][0][0]} -- Operator' + + resolved = view["resolve_support"]({"cited": [citation]}) + + assert resolved == {"cited": [f"{event_id} - Operator"]} + assert validate_support( + resolved, + set(view["canonical_nodes"]), + set(view["canonical_edges"]), + ) == resolved + + +def test_event_join_skips_missing_anchor_and_yields_valid_anchored_event(): + from graphgen.models.partitioner.event_join_partitioner import EventJoinPartitioner + + class FakeStorage: + def get_all_nodes(self): + return [ + ("event:unanchored", {"entity_type": "EVENT", "graph_role": "event", "anchor": "", "length": 1}), + ("event:anchored", {"entity_type": "EVENT", "graph_role": "event", "anchor": "CASE-1", "length": 1}), + ("entity:operator", {"entity_type": "ACTOR", "length": 1}), + ] + + def get_neighbors(self, node_id): + return ["entity:operator"] if node_id == "event:anchored" else [] + + communities = list(EventJoinPartitioner().partition(FakeStorage(), require_same_anchor=True)) + + assert len(communities) == 1 + assert communities[0].id == "event:anchored" + assert "event:unanchored" not in communities[0].nodes + + + + +def test_schema_guided_extractor_uses_json_compatible_prefill(monkeypatch): + import asyncio + from types import SimpleNamespace + + from graphgen.bases.datatypes import Chunk + from graphgen.models.extractor.schema_guided_extractor import SchemaGuidedExtractor + + class FakeLLM: + def __init__(self): + self.calls = [] + + async def generate_answer(self, text, **kwargs): + self.calls.append(kwargs.get("assistant_prefill")) + return "" if len(self.calls) == 1 else '{"name":"value"}' + + monkeypatch.setenv("GRAPHGEN_EMPTY_EXTRACTION_PREFILL", "1") + llm = FakeLLM() + extractor = SchemaGuidedExtractor(llm, {"name": "item", "properties": {"name": {"description": "name"}}, "required": ["name"]}) + result = asyncio.run(extractor.extract(Chunk(id="chunk-1", content="a short text", type="text"))) + + assert result == {"name": "value"} + assert llm.calls == [None, "{"] + + +def test_openai_empty_retry_reapplies_rpm_and_tpm_limits(monkeypatch): + import asyncio + import types + + from graphgen.models.llm.api.openai_client import OpenAIClient + + class Limiter: + def __init__(self): + self.calls = [] + + async def wait(self, *args, **kwargs): + self.calls.append(args) + + class Completion: + def __init__(self, content): + self.choices = [types.SimpleNamespace(message=types.SimpleNamespace(content=content))] + self.usage = types.SimpleNamespace(prompt_tokens=1, completion_tokens=1, total_tokens=2) + + class Completions: + def __init__(self): + self.responses = iter([Completion(""), Completion("ok")]) + + async def create(self, **_kwargs): + return next(self.responses) + + client = object.__new__(OpenAIClient) + client._pre_generate = lambda *_args, **_kwargs: {"messages": [{"content": "x"}], "max_tokens": 4} + client.tokenizer = types.SimpleNamespace(encode=lambda _text: [1]) + client.request_limit = True + client.rpm, client.tpm = Limiter(), Limiter() + client.model = "test" + client.token_usage = [] + client.client = types.SimpleNamespace(chat=types.SimpleNamespace(completions=Completions())) + monkeypatch.setenv("GRAPHGEN_EMPTY_RETRY", "1") + + assert asyncio.run(client.generate_answer("prompt")) == "ok" + assert len(client.rpm.calls) == 2 + assert len(client.tpm.calls) == 2 diff --git a/tests/test_support_edge_citations.py b/tests/test_support_edge_citations.py new file mode 100644 index 00000000..8aa9378c --- /dev/null +++ b/tests/test_support_edge_citations.py @@ -0,0 +1,49 @@ +from graphgen.models.generator.support import build_generation_view, validate_support + + +def _item(): + event_id = "event:opaque-hash" + return { + "nodes": [ + (event_id, {"entity_type": "EVENT", "graph_role": "event", "description": "Migration completed：fact"}), + ("Operator", {"entity_type": "ACTOR", "description": "responsible"}), + ], + "edges": [(event_id, "Operator", {"description": "participated"})], + } + + +def test_rendered_edge_alias_resolves_and_only_present_canonical_edges_validate(): + item = _item() + view = build_generation_view(item) + citation = f'{view["nodes"][0][0]} -- Operator' + resolved = view["resolve_support"]({"cited": [citation]}) + known_edge = "event:opaque-hash - Operator" + + assert resolved == {"cited": [known_edge]} + assert validate_support( + resolved, + set(view["canonical_nodes"]), + set(view["canonical_edges"]), + ) == resolved + assert validate_support( + {"cited": ["event:opaque-hash - Missing"]}, + set(view["canonical_nodes"]), + set(view["canonical_edges"]), + ) is None + + +def test_rendered_edge_alias_resolves_when_partition_reverses_edge_endpoints(): + item = _item() + event_id = item["nodes"][0][0] + item["edges"] = [("Operator", event_id, {"description": "participated"})] + view = build_generation_view(item) + citation = f'Operator -- {view["nodes"][0][0]}' + + resolved = view["resolve_support"]({"cited": [citation]}) + + assert resolved == {"cited": [f"Operator - {event_id}"]} + assert validate_support( + resolved, + set(view["canonical_nodes"]), + set(view["canonical_edges"]), + ) == resolved From 3a2012d30283913d0d0258f7a8316484c7d95f03 Mon Sep 17 00:00:00 2001 From: Leon-L20 Date: Fri, 16 Oct 2026 09:29:22 +0000 Subject: [PATCH 8/9] =?UTF-8?q?docs:=20=E8=AE=B0=E5=BD=95=20GraphGen=201.1?= =?UTF-8?q?.2=20=E5=8F=91=E8=A1=8C=E4=BA=A7=E7=89=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 登记最终候选提交、wheel/sdist 哈希及 Python 3.12 安装证据。 Co-Authored-By: Claude Opus 5.5 --- docs/RELEASE_MANIFEST-1.1.2.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/RELEASE_MANIFEST-1.1.2.md b/docs/RELEASE_MANIFEST-1.1.2.md index e31233bc..78e5a466 100644 --- a/docs/RELEASE_MANIFEST-1.1.2.md +++ b/docs/RELEASE_MANIFEST-1.1.2.md @@ -1,6 +1,6 @@ # GraphGen 1.1.2 release manifest -Status: validation evidence for the release candidate; commit SHA is recorded after final release commit. +Status: final release candidate built from commit `49ec1ac8dfe9c15204f979c49887f5614a5adf6d`; publication record remains pending. ## Source identity @@ -35,7 +35,9 @@ Built in a disposable output directory with `uv build --offline --out-dir /tmp/g ## Publication record -- Final commit SHA: pending +- Final commit SHA: `49ec1ac8dfe9c15204f979c49887f5614a5adf6d` +- Final commit wheel: `graphg-1.1.2-py3-none-any.whl` — SHA-256 `084ae2a0cf0b5afd111fd74e106295face1f892def587db3fca5fc620e1b092b` +- Final commit sdist: `graphg-1.1.2.tar.gz` — SHA-256 `9f83b6f7565339a4ae4379fc8176807dfa28fc0684651dae381472217dc740c3` - Tag creation: pending; must verify local and `fork` remote tag absence first. - Fork branch/tag push: pending maintainer action after final diff review. - PMS dependency pin update and locked sync: pending 1.1.2 commit SHA after publication. From d45b17be82b52da21daa9a8ffe813a303604c698 Mon Sep 17 00:00:00 2001 From: Leon-L20 Date: Fri, 16 Oct 2026 09:57:15 +0000 Subject: [PATCH 9/9] =?UTF-8?q?docs:=20=E6=94=B6=E5=8F=A3=20GraphGen=201.1?= =?UTF-8?q?.2=20=E5=8F=91=E5=B8=83=E9=AA=8C=E6=94=B6=E8=AE=B0=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 记录 fork tag、PMSAgent pin、锁文件同步与兼容测试最终结果。 Co-Authored-By: Claude Opus 5.5 --- docs/RELEASE_MANIFEST-1.1.2.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/RELEASE_MANIFEST-1.1.2.md b/docs/RELEASE_MANIFEST-1.1.2.md index 78e5a466..dcdebb79 100644 --- a/docs/RELEASE_MANIFEST-1.1.2.md +++ b/docs/RELEASE_MANIFEST-1.1.2.md @@ -1,6 +1,6 @@ # GraphGen 1.1.2 release manifest -Status: final release candidate built from commit `49ec1ac8dfe9c15204f979c49887f5614a5adf6d`; publication record remains pending. +Status: released and deployed to PMS production pin; final release record is documented below. ## Source identity @@ -16,10 +16,13 @@ Run in the GraphGen defect worktree with Python 3.12 dependencies supplied by `w - `PYTHONPATH=: UV_OFFLINE=1 NLTK_DISABLE_IMPORT_SECURITY=1 -m pytest -c /dev/null -q tests/unit_tests tests/test_event_profile_loader_regressions.py tests/test_support_edge_citations.py tests/test_event_profiles.py` — **15 passed**. - `PYTHONPATH=: UV_OFFLINE=1 NLTK_DISABLE_IMPORT_SECURITY=1 -m pytest -c /dev/null -q tests/test_package_profiles.py` — **1 passed**. -- `PYTHONPATH=: NLTK_DISABLE_IMPORT_SECURITY=1 -m pytest -c /dev/null -q /tests/test_pms_graphgen_policy_generation.py /tests/test_run_graphgen_defaults.py /tests/test_repair_project_query_graphgen_chunk_cache.py` — **28 passed, 1 skipped**. +- `PYTHONPATH=: NLTK_DISABLE_IMPORT_SECURITY=1 -m pytest -c /dev/null -q /tests/test_graphgen_production_source.py /tests/test_pms_graphgen_policy_generation.py /tests/test_run_graphgen_defaults.py /tests/test_repair_project_query_graphgen_chunk_cache.py` — latest PMSAgent line **33 passed, 1 skipped**. - `PYTHONPATH=: NLTK_DISABLE_IMPORT_SECURITY=1 -m pytest -c /dev/null -q tests/integration_tests/models/partitioner/test_ece_partitioner.py` — **6 passed**. - Non-PMS deterministic integration smoke via `EventEntityKGBuilder` extraction+merge → `EventJoinPartitioner`: **3 stub LLM calls, 3 event nodes, 3 membership edges, 3 communities; role persisted on every edge; anchors CASE-A/CASE-B/CASE-C; no network/model calls**. Executed by the GraphGen test agent; command: `PYTHONPATH= /.venv/bin/python /tmp/graphgen_event_join_smoke.py`. - `git diff --check` — passed. +- PMS production dependency pin and lock resolved to GraphGen commit `49ec1ac8dfe9c15204f979c49887f5614a5adf6d`; `uv lock --locked --offline` and `uv sync --locked --offline` passed. Production venv reports `graphg==1.1.2`, loaded from its `site-packages`. +- PMSAgent final commit `17ea5674fda776116b91cf86ebe91c599488550b`; verified on `origin/PMSAgent`. +- Upstream `origin` and PR #192 were not modified. ## Build and clean-install evidence @@ -35,9 +38,9 @@ Built in a disposable output directory with `uv build --offline --out-dir /tmp/g ## Publication record -- Final commit SHA: `49ec1ac8dfe9c15204f979c49887f5614a5adf6d` +- GraphGen fix commit SHA: `49ec1ac8dfe9c15204f979c49887f5614a5adf6d` +- Published fork branch `pms/main` doc-only tip: `3a2012d30283913d0d0258f7a8316484c7d95f03`. +- Immutable annotated tag `graphgen-fork-v1.1.2` peels to the fix commit SHA above. - Final commit wheel: `graphg-1.1.2-py3-none-any.whl` — SHA-256 `084ae2a0cf0b5afd111fd74e106295face1f892def587db3fca5fc620e1b092b` - Final commit sdist: `graphg-1.1.2.tar.gz` — SHA-256 `9f83b6f7565339a4ae4379fc8176807dfa28fc0684651dae381472217dc740c3` -- Tag creation: pending; must verify local and `fork` remote tag absence first. -- Fork branch/tag push: pending maintainer action after final diff review. -- PMS dependency pin update and locked sync: pending 1.1.2 commit SHA after publication. +- PMS dependency pin and locked sync are complete; PMSAgent commit `17ea5674fda776116b91cf86ebe91c599488550b` contains the new pin.