diqduq

50class Token(BaseModel):
51    """A single pre-segmented token with a stable id.
52
53    `citation` is optional so this model still works for citation-free
54    callers -- e.g. a test fixture built directly from a canned tokengraph,
55    with no CitedText source at all -- as well as for the citation-aware
56    segmentation stage (segmentation_dspy.py), which is the only thing that
57    actually populates it, knowing which CitedText source unit each token
58    came from."""
59
60    id: str = Field(description="Stable token id, globally unique and sequential across the whole input, e.g. 't0', 't1', ...")
61    text: str = Field(description="The token's surface text, exactly as it appears in the source.")
62    citation: Optional[str] = Field(
63        default=None,
64        description="Citation label of the source unit this token came from (e.g. 'urn:cts:compnov:bible.genesis.masoretic:1.1'), if known.",
65    )

39class CitedText(BaseModel):
40    """One citable unit of source text -- e.g. one verse -- paired with its
41    citation label. A sequence of these is segmentation_dspy.py's input:
42    sentence boundaries do NOT need to respect CitedText boundaries (one
43    sentence may span several units), but every resulting token still
44    records which unit it came from via Token.citation."""
45
46    citation: str = Field(description="Citation label for this unit, e.g. 'urn:cts:compnov:bible.genesis.masoretic:1.1'.")
47    text: str = Field(description="This unit's raw text, exactly as written (including niqqud and cantillation).")

68class Sentence(BaseModel):
69    """One sentence's worth of tokens, in reading order, as produced by the
70    LLM-driven segmentation stage (segmentation_dspy.py). Token ids are
71    global across the whole passage -- numbering continues across sentence
72    boundaries rather than restarting at t0 for each sentence -- so a
73    Sentence is a contiguous slice of the passage's id sequence, not an
74    independently-numbered unit."""
75
76    tokens: List[Token] = Field(
77        description="This sentence's tokens, in reading order, using the passage's global token ids."
78    )

 81class VerbalExpression(BaseModel):
 82    """One entry in the table of verbal expressions (syntax_model.md,
 83    'Table of verbal expressions'). Two constructions count as a verbal
 84    expression, in this first draft of the scheme:
 85
 86    1. Every finite verb.
 87    2. Every participle.
 88
 89    Each verbal expression is classified along two independent axes:
 90
 91    - `syntactic_type`: 'independent' (also called "main" or "principal" --
 92      a syntactically independent finite verb, whose clause is coherent by
 93      itself) or 'direct quote' (a verbal expression occurring in directly
 94      quoted speech -- see RelationLabel's 'direct quote' value for how it
 95      relates back to the verb that introduces the quotation). These are
 96      the only two values syntax_model.md currently documents; unlike
 97      arsgrammatica's mature Latin scheme, there is no 'dependent' category
 98      yet (subordinating conjunctions are listed under syntax_model.md's
 99      'TBA' section) and no separate category for an "aside" or an
100      "indirect statement" -- Biblical Hebrew narrative marks reported
101      speech directly (see 'direct quote' below) rather than through an
102      accusative-and-infinitive construction the way Latin does.
103    - `semantic_type`: 'transitive active', 'transitive passive',
104      'intransitive', or 'linking verb'.
105
106    A participle's own syntactic_type isn't pinned down by syntax_model.md
107    beyond the two values above -- this codebase's convention (flagged
108    here, matching the style of arsgrammatica's own documented judgment
109    calls, since syntax_model.md itself doesn't say) is to classify a
110    participle 'independent' unless it occurs within quoted speech, in
111    which case 'direct quote' applies exactly as it would to a finite verb.
112    Extend syntax_model.md first if a real passage needs a genuinely
113    different category for a participial clause (e.g. something
114    circumstantial/subordinate, as arsgrammatica's Latin scheme has for a
115    circumstantial participle) -- see notes/USAGE.md's "Extending the scheme".
116
117    `diqduq` also recognizes one construction as understood or implied
118    even though it has no surface realization: an elided present of "to
119    be" (see IMPLIED_TOKENTYPES's 'implied sum' below, and TokenAnalysis's
120    own docstring for the new-token convention this requires)."""
121
122    id: str = Field(
123        description=(
124            "The token id (from the input `tokens` list) of the finite "
125            "verb or participle that anchors this verbal expression. For "
126            "an elided present of 'to be' (see TokenAnalysis's 'implied "
127            "sum' tokentype, IMPLIED_TOKENTYPES), use the new implied "
128            "token's id instead."
129        )
130    )
131    syntactic_type: Literal["independent", "direct quote"] = Field(
132        description=(
133            "'independent' (main/principal -- a syntactically independent "
134            "finite verb or participle) or 'direct quote' (occurring in "
135            "directly quoted speech; see RelationLabel's 'direct quote' "
136            "value). syntax_model.md documents no other values yet -- see "
137            "this model's own docstring."
138        )
139    )
140    semantic_type: Literal[
141        "transitive active", "transitive passive", "intransitive", "linking verb"
142    ] = Field(description="The verb's semantic/voice type.")

264class TokenAnalysis(BaseModel):
265    """One entry per token in the dependency graph (syntax_model.md,
266    'Token-level table of dependencies'). Per syntax_model.md's own
267    "TBA" section (some constructions have no documented relation at all
268    yet), not every token will have a relation -- leave the
269    relatedtoken*/relationship* fields unset when none of the documented
270    relations apply.
271
272    Every entry corresponds 1:1 to an entry in the input `tokens` list,
273    EXCEPT for the one IMPLIED_TOKENTYPES value below: syntax_model.md's
274    "understood or implied verbal expressions" section documents an elided
275    present of "to be" ("to be" is often left out of a Hebrew nominal
276    sentence entirely) as a VERBAL EXPRESSION that exists grammatically
277    but has no surface realization at all. For that case, add a NEW entry
278    here -- with a NEW id, not present in `tokens` -- rather than skipping
279    the construction: tokentype 'implied sum', with `token` left unset
280    (None). See hebrew_syntax_dspy.SyntaxAnalysis's docstring for the full
281    rule and the id-naming convention."""
282
283    id: str = Field(
284        description=(
285            "For an ordinary entry, must match the id of the corresponding "
286            "entry in the input `tokens` list. For an implied token "
287            "(tokentype 'implied sum'), a NEW id not used by any entry in "
288            "`tokens` or elsewhere in this tokengraph -- see "
289            "SyntaxAnalysis's docstring for the naming convention."
290        )
291    )
292    token: Optional[str] = Field(
293        default=None,
294        description=(
295            "The token's surface text; should match the `text` of the "
296            "input token with this id. Leave as None ONLY for an implied "
297            "token (tokentype 'implied sum') -- one with no surface "
298            "realization in the passage at all; every other tokentype "
299            "must have real text."
300        ),
301    )
302    tokentype: Literal[
303        "lexical",
304        "enclitic pronoun",
305        "proclitic conjunction",
306        "maqaf",
307        "cantillation",
308        "paragraph",
309        "editorial",
310        "implied sum",
311    ] = Field(
312        description=(
313            "Per syntax_model.md's 'Tokenization' section: 'cantillation' "
314            "for any of the te'amim (e.g. sof pasuq, silluq, atnach); "
315            "'paragraph' for a פ (petuhah) or ס (setumah) marking a "
316            "semantic division of the text; 'enclitic pronoun' for a "
317            "pronoun bound as the object of a preposition or verb, or as a "
318            "possessive with a noun; 'proclitic conjunction' specifically "
319            "for the conjunction וְ; 'maqaf' for the joining token ־; "
320            "'lexical' for a continuous alphabetic sequence together with "
321            "its own niqqud/dagesh/mappiq/sin-shin-dot (but never "
322            "cantillation marks, which are their own token type); "
323            "'editorial' for any Unicode punctuation character or other "
324            "editorial mark, such as the masora circle. 'implied sum' "
325            "marks a token with NO surface realization at all (an elided "
326            "present of 'to be' -- see this model's own docstring) -- the "
327            "only tokentype whose `token` field is None and whose `id` is "
328            "not one of the input `tokens`' own ids; it always anchors its "
329            "own entry in `verbalunits`, exactly like a real verb."
330        )
331    )
332
333    lemma: Optional[str] = Field(default=None, description="Dictionary headword, for lexical tokens. Omit for cantillation/paragraph/editorial tokens.")
334    verbalunitid: Optional[str] = Field(
335        default=None,
336        description="If this token anchors a verbal expression in `verbalunits`, repeat its own id here; otherwise omit.",
337    )
338
339    relatedtoken1: Optional[str] = Field(
340        default=None,
341        description=(
342            "Token id this token relates to (primary relation). For an "
343            "INDEPENDENT verb's own 'unit verb' relation, use the special "
344            "sentinel string 'root' instead of a token id -- 'root' is "
345            "reserved and must never be assigned as an actual token's id."
346        ),
347    )
348    relationship1: Optional[RelationLabel] = Field(default=None, description="The primary relation type, if any.")
349
350    relatedtoken2: Optional[str] = Field(default=None, description="Token id this token relates to (secondary relation -- an overflow slot, EXCEPT for 'coordinating conjunction', which uses both slots for its two sides at once).")
351    relationship2: Optional[RelationLabel] = Field(default=None, description="The secondary relation type, if any.")

 70def tokengraph_to_mermaid(
 71    tokengraph: List[TokenAnalysis],
 72    orientation: str = "BT",
 73    color_by_verbal_unit: bool = True,
 74    rank_by_depth: bool = True,
 75) -> Tuple[str, List[str]]:
 76    """Build a Mermaid `graph` diagram from a tokengraph.
 77
 78    `orientation` is Mermaid's own flowchart orientation code -- `BT`
 79    (bottom-to-top, the default here), `TB`, `LR`, or `RL` -- used verbatim
 80    in the diagram's opening line (`graph BT`, `graph LR`, etc.). See
 81    https://mermaid.js.org/syntax/flowchart.html for what each value looks
 82    like.
 83
 84    `color_by_verbal_unit` (default True) colors every node by the verbal
 85    unit it belongs to, per verbal_units.assign_verbal_units(). Pass False
 86    to skip coloring and get a plain diagram.
 87
 88    `rank_by_depth` (default True) makes the diagram's layout respect each
 89    verbal expression's own *depth of subordination* (see
 90    verbal_units.compute_subordination_depths()). Pass False to skip this
 91    and get the diagram's previous, unranked layout.
 92
 93    Returns (diagram_text, warnings). `warnings` lists any edges that were
 94    skipped because they referenced a non-substantive token or an id not
 95    present in `tokengraph`, plus, if `color_by_verbal_unit` is True and the
 96    passage has more than 8 verbal units, one warning that colors are
 97    repeating, plus, if `rank_by_depth` is True, any of
 98    compute_subordination_depths()'s own warnings.
 99    """
100    node_ids = {tok.id for tok in tokengraph if tok.tokentype not in NON_SUBSTANTIVE_TOKENTYPES}
101
102    lines = [f"graph {orientation}"]
103    for tok in tokengraph:
104        if tok.id not in node_ids:
105            continue
106        label = (
107            tok.token
108            if tok.token is not None
109            else _IMPLIED_TOKEN_LABELS.get(tok.tokentype, tok.tokentype)
110        )
111        open_bracket, close_bracket = (
112            ("(", ")") if tok.tokentype in IMPLIED_TOKENTYPES else ("[", "]")
113        )
114        lines.append(f'    {tok.id}{open_bracket}"{_escape_label(label)}"{close_bracket}')
115
116    warnings = []
117    for tok in tokengraph:
118        if tok.id not in node_ids:
119            continue
120        for related_field, label_field in (
121            ("relatedtoken1", "relationship1"),
122            ("relatedtoken2", "relationship2"),
123        ):
124            related_id = getattr(tok, related_field)
125            label = getattr(tok, label_field)
126            if related_id is None or label is None:
127                continue
128            if related_id == "root":
129                continue
130            if related_id not in node_ids:
131                warnings.append(
132                    f"skipped edge {tok.id} -[{label}]-> {related_id}: "
133                    f"target is non-substantive or not in tokengraph"
134                )
135                continue
136            lines.append(f'    {tok.id} -->|{_escape_label(label)}| {related_id}')
137
138    if rank_by_depth:
139        depths, depth_warnings = compute_subordination_depths(tokengraph)
140        warnings.extend(depth_warnings)
141
142        depth_groups: dict = {}
143        for tok in tokengraph:
144            if tok.id not in node_ids:
145                continue
146            depth = depths.get(tok.id)
147            if depth is None:
148                continue
149            depth_groups.setdefault(depth, []).append(tok.id)
150
151        rank_lines = [
152            "    " + " ~~~ ".join(ids)
153            for depth in sorted(depth_groups)
154            for ids in (depth_groups[depth],)
155            if len(ids) > 1
156        ]
157        if rank_lines:
158            lines.append("")
159            lines.extend(rank_lines)
160
161    if color_by_verbal_unit:
162        assignment = assign_verbal_units(tokengraph)
163        colors, color_warnings = assign_verbal_unit_colors(tokengraph, assignment=assignment)
164        warnings.extend(color_warnings)
165
166        implied_ids = [
167            tok.id
168            for tok in tokengraph
169            if tok.id in node_ids and tok.tokentype in IMPLIED_TOKENTYPES
170        ]
171
172        if colors or implied_ids:
173            lines.append("")
174            class_names = {}
175            for i, (unit_id, (fill, stroke, text)) in enumerate(colors.items()):
176                class_name = f"vu{i}"
177                class_names[unit_id] = class_name
178                lines.append(
179                    f"    classDef {class_name} fill:{fill},stroke:{stroke},color:{text};"
180                )
181            for unit_id in colors:
182                member_ids = [
183                    tok.id
184                    for tok in tokengraph
185                    if tok.id in node_ids
186                    and assignment.get(tok.id) == unit_id
187                    and tok.id not in implied_ids
188                ]
189                if member_ids:
190                    lines.append(f"    class {','.join(member_ids)} {class_names[unit_id]};")
191            if implied_ids:
192                fill, stroke, text = _IMPLIED_TOKEN_COLOR
193                lines.append(
194                    f"    classDef implied fill:{fill},stroke:{stroke},color:{text};"
195                )
196                lines.append(f"    class {','.join(implied_ids)} implied;")
197
198    return "\n".join(lines), warnings

201def save_mermaid(
202    tokengraph: List[TokenAnalysis],
203    path: str,
204    orientation: str = "BT",
205    color_by_verbal_unit: bool = True,
206    rank_by_depth: bool = True,
207) -> List[str]:
208    """Write the diagram to `path` (e.g. 'analysis.mmd') and return any
209    warnings from tokengraph_to_mermaid."""
210    diagram, warnings = tokengraph_to_mermaid(
211        tokengraph,
212        orientation=orientation,
213        color_by_verbal_unit=color_by_verbal_unit,
214        rank_by_depth=rank_by_depth,
215    )
216    with open(path, "w", encoding="utf-8") as f:
217        f.write(diagram + "\n")
218    return warnings

 73def assign_verbal_units(tokengraph: List[TokenAnalysis]) -> Dict[str, Optional[str]]:
 74    """Return {token id: verbal unit id or None}, one entry per token in
 75    `tokengraph` (including cantillation/editorial/maqaf and any other
 76    unrelated token, so every id is accounted for -- callers that only
 77    care about assigned tokens can filter out the None values themselves).
 78
 79    A verbal unit's own anchor token is assigned to itself (its
 80    `verbalunitid`). Every other token is assigned to the verbal unit its
 81    relations resolve to via a plain forward chase of relatedtoken1 (then
 82    relatedtoken2); a token with no resolvable relation (e.g. a
 83    preposition, whose own outward relation syntax_model.md doesn't
 84    document yet -- see this module's own docstring) gets None.
 85    """
 86    by_id = {tok.id: tok for tok in tokengraph}
 87
 88    resolved: Dict[str, Optional[str]] = {}
 89    in_progress: set = set()
 90
 91    def resolve(tid: str) -> Optional[str]:
 92        if tid in resolved:
 93            return resolved[tid]
 94        tok = by_id.get(tid)
 95        if tok is None:
 96            return None
 97
 98        if tok.verbalunitid is not None:
 99            resolved[tid] = tok.verbalunitid
100            return tok.verbalunitid
101
102        if tid in in_progress:
103            # A cycle in the relation graph (malformed LM output) -- bail
104            # out on this token rather than recursing forever.
105            return None
106        in_progress.add(tid)
107
108        result = None
109        for related_field in ("relatedtoken1", "relatedtoken2"):
110            related = getattr(tok, related_field)
111            if related is None or related == "root":
112                continue
113            result = resolve(related)
114            if result is not None:
115                break
116
117        in_progress.discard(tid)
118        resolved[tid] = result
119        return result
120
121    for tid in by_id:
122        resolve(tid)
123
124    return resolved

127def assign_verbal_unit_colors(
128    tokengraph: List[TokenAnalysis],
129    assignment: Optional[Dict[str, Optional[str]]] = None,
130) -> Tuple[Dict[str, Tuple[str, str, str]], List[str]]:
131    """Assign each verbal unit found in `tokengraph` a stable (fill, stroke,
132    text) triple from `_VERBAL_UNIT_PALETTE`, using the exact ordering rule
133    `tokengraph_to_mermaid()` uses for its node coloring -- so any other
134    caller wanting "the same colors as the mermaid graph" (rendering.py's
135    `tokengraph_to_html()`) gets an identical mapping without re-deriving
136    the rule itself.
137
138    Order is by first appearance of each verbal unit among tokengraph's
139    *substantive* tokens (tokentype not in models.NON_SUBSTANTIVE_TOKENTYPES
140    -- cantillation, paragraph, editorial, and maqaf never become mermaid
141    nodes at all).
142
143    Pass `assignment` (the result of `assign_verbal_units(tokengraph)`) if
144    the caller already computed it, to avoid re-deriving it here; otherwise
145    it's computed internally.
146
147    Returns `({verbal unit id: (fill, stroke, text)}, warnings)` --
148    `warnings` holds one entry if there are more distinct verbal units than
149    palette slots (colors repeat past the 8th unit). A verbal unit id
150    absent from the returned dict was never assigned to any substantive
151    token -- callers should treat that the same as "no verbal unit" (no
152    coloring).
153    """
154    if assignment is None:
155        assignment = assign_verbal_units(tokengraph)
156
157    substantive_ids = {tok.id for tok in tokengraph if tok.tokentype not in NON_SUBSTANTIVE_TOKENTYPES}
158
159    unit_order: List[str] = []
160    seen_units = set()
161    for tok in tokengraph:
162        if tok.id not in substantive_ids:
163            continue
164        unit_id = assignment.get(tok.id)
165        if unit_id is not None and unit_id not in seen_units:
166            seen_units.add(unit_id)
167            unit_order.append(unit_id)
168
169    warnings: List[str] = []
170    if len(unit_order) > len(_VERBAL_UNIT_PALETTE):
171        warnings.append(
172            f"{len(unit_order)} verbal units but only {len(_VERBAL_UNIT_PALETTE)} "
173            "distinct colors -- colors repeat and may be ambiguous between units"
174        )
175
176    colors = {
177        unit_id: _VERBAL_UNIT_PALETTE[i % len(_VERBAL_UNIT_PALETTE)]
178        for i, unit_id in enumerate(unit_order)
179    }
180    return colors, warnings

183def compute_subordination_depths(
184    tokengraph: List[TokenAnalysis],
185) -> Tuple[Dict[str, Optional[int]], List[str]]:
186    """Compute each verbal expression's *depth of subordination*: the
187    number of verbal expressions it is removed from an independent ("root")
188    clause. An independent verb is depth 0; a directly-quoted verb it
189    introduces is depth 1; a verbal expression quoted WITHIN that quote (if
190    the scheme is ever asked to represent one) would be depth 2; and so on.
191
192    A "verbal expression" here is any token that anchors one -- i.e. any
193    token with `verbalunitid` set to its own id (the same convention
194    `assign_verbal_units()` relies on). For each anchor, this function
195    finds its *parent* anchor -- the verbal expression it's subordinate to
196    -- by following the anchor's own relatedtoken1 (falling back to
197    relatedtoken2) until it lands on another anchor:
198
199    - unit verb (independent): relatedtoken1 == 'root' -> no parent, depth 0.
200    - direct quote: relatedtoken1 -> the verb of the clause that introduces
201      the quotation, directly (no intermediate token to hop through, unlike
202      arsgrammatica's Latin scheme, which has no equivalent of a
203      subordinating-conjunction intermediary for this case either).
204
205    Returns `({anchor id: depth or None}, warnings)`. A depth of `None`
206    means the chase from that anchor never reached another anchor (a
207    malformed or genuinely disconnected verbal expression) or a cycle was
208    detected; `warnings` names which anchor(s) and why, rather than
209    raising.
210    """
211    by_id = {tok.id: tok for tok in tokengraph}
212    anchor_ids = {tok.id for tok in tokengraph if tok.verbalunitid == tok.id}
213
214    warnings: List[str] = []
215
216    def chase(token_id: str, visited: set) -> Optional[str]:
217        """Follow relatedtoken1 (then relatedtoken2) forward from
218        `token_id`, returning the first anchor id reached, or None if the
219        chain dead-ends or cycles before reaching one. `token_id` itself
220        counts as a hit if it's already an anchor (the direct 'direct
221        quote' case)."""
222        if token_id in visited:
223            return None
224        visited.add(token_id)
225        if token_id in anchor_ids:
226            return token_id
227        tok = by_id.get(token_id)
228        if tok is None:
229            return None
230        for field in ("relatedtoken1", "relatedtoken2"):
231            target = getattr(tok, field)
232            if target is None or target == "root":
233                continue
234            result = chase(target, visited)
235            if result is not None:
236                return result
237        return None
238
239    def parent_of(anchor_id: str) -> Optional[str]:
240        tok = by_id[anchor_id]
241        for field in ("relatedtoken1", "relatedtoken2"):
242            target = getattr(tok, field)
243            if target is None or target == "root":
244                continue
245            result = chase(target, visited=set())
246            if result is not None and result != anchor_id:
247                return result
248        return None
249
250    depths: Dict[str, Optional[int]] = {}
251    in_progress: set = set()
252
253    def depth_of(anchor_id: str) -> Optional[int]:
254        if anchor_id in depths:
255            return depths[anchor_id]
256        tok = by_id[anchor_id]
257        if tok.relatedtoken1 == "root":
258            depths[anchor_id] = 0
259            return 0
260
261        if anchor_id in in_progress:
262            warnings.append(
263                f"cycle detected resolving the governing verbal expression "
264                f"for {anchor_id!r} -- leaving its depth (and its parent's) "
265                f"unresolved"
266            )
267            return None
268        in_progress.add(anchor_id)
269
270        parent = parent_of(anchor_id)
271        if parent is None:
272            warnings.append(
273                f"could not find a governing verbal expression for "
274                f"{anchor_id!r} -- leaving its depth unresolved"
275            )
276            result = None
277        else:
278            parent_depth = depth_of(parent)
279            result = None if parent_depth is None else parent_depth + 1
280
281        in_progress.discard(anchor_id)
282        depths[anchor_id] = result
283        return result
284
285    for anchor_id in anchor_ids:
286        depth_of(anchor_id)
287
288    return depths, warnings

291def max_subordination_depth(
292    tokengraph: List[TokenAnalysis],
293    depths: Optional[Dict[str, Optional[int]]] = None,
294) -> Optional[int]:
295    """Return the deepest level of subordination reached anywhere in
296    `tokengraph` -- the highest value `compute_subordination_depths()`
297    assigns to any verbal expression. Root/independent clauses are depth
298    0, so this is also the upper end of the valid `depth` range for
299    `rendering.tokengraph_to_depth_html()`'s own `depth` parameter.
300
301    Pass `depths` (the first element of `compute_subordination_depths()`'s
302    return value) if the caller already computed it, to avoid re-deriving
303    it here.
304
305    Returns `None` if `tokengraph` has no verbal expressions at all, or if
306    every anchor's own depth came back unresolved. Otherwise returns the
307    maximum of every RESOLVED anchor's depth, ignoring unresolved ones
308    rather than letting a single bad anchor blank out the whole result.
309    """
310    if depths is None:
311        depths, _warnings = compute_subordination_depths(tokengraph)
312
313    resolved = [d for d in depths.values() if d is not None]
314    if not resolved:
315        return None
316    return max(resolved)

319def find_unanchored_coordinated_verbs(tokengraph: List[TokenAnalysis]) -> List[str]:
320    """Heuristic sanity check for a specific, plausible live-LM mistake: a
321    coordinating conjunction that pairs two verbal expressions (see
322    hebrew_syntax_dspy.py's docstring) is supposed to leave BOTH conjuncts
323    anchoring their own verbal unit -- each with its own `verbalunitid`
324    (and its own `verbalunits` entry). This is NOT the same kind of check
325    as validate() (referential id integrity) or
326    compute_subordination_depths()'s warnings (a resolvable-but-broken
327    relation graph) -- both of those only catch a problem if the
328    tokengraph is already self-inconsistent. This function catches a
329    tokengraph that's perfectly well-formed and internally consistent, but
330    still probably WRONG, by looking for an asymmetry a correct analysis
331    should never produce.
332
333    The heuristic: find every "coordinating conjunction" token that uses
334    BOTH relatedtoken1 and relatedtoken2 (the two-conjunct, single-pair
335    case -- see that relation's own note about the repeated-connector,
336    series exception, which this deliberately ignores below). For each
337    such pair, if EXACTLY ONE of the two joined tokens is a recognized
338    verbal-unit anchor (`verbalunitid` set to its own id) and the other is
339    not, that asymmetry is flagged: if the conjunction is genuinely pairing
340    two nouns/adjectives/prepositional phrases, NEITHER side would be an
341    anchor; if it's correctly pairing two verbal expressions, BOTH sides
342    would be.
343
344    This pairwise shape doesn't apply to a repeated connector coordinating
345    a series of two or more items via the id-chaining convention (see
346    hebrew_syntax_dspy.py's docstring): there, every connector's own
347    relatedtoken2 points at a NEIGHBORING CONNECTOR, not at a second
348    conjunct, so this heuristic's asymmetry check would misfire. A pair is
349    therefore skipped whenever relatedtoken2 resolves to a token that is
350    itself a coordinating-conjunction connector.
351
352    Returns a list of warning strings (empty if nothing looks suspicious).
353    This is a heuristic, not a guarantee.
354    """
355    by_id = {tok.id: tok for tok in tokengraph}
356    anchor_ids = {tok.id for tok in tokengraph if tok.verbalunitid == tok.id}
357
358    warnings: List[str] = []
359    seen_pairs = set()
360
361    for tok in tokengraph:
362        if not (
363            tok.relatedtoken1 is not None
364            and tok.relatedtoken1 != "root"
365            and tok.relationship1 == "coordinating conjunction"
366            and tok.relatedtoken2 is not None
367            and tok.relatedtoken2 != "root"
368            and tok.relationship2 == "coordinating conjunction"
369        ):
370            continue
371
372        pair = (tok.relatedtoken1, tok.relatedtoken2)
373        if pair in seen_pairs:
374            continue
375        seen_pairs.add(pair)
376
377        first_id, second_id = pair
378        second_tok = by_id.get(second_id)
379        if second_tok is not None and (
380            second_tok.relationship1 == "coordinating conjunction"
381            or second_tok.relationship2 == "coordinating conjunction"
382        ):
383            # relatedtoken2 points at a FELLOW connector, not at a second
384            # conjunct -- the signature of the repeated-connector/series
385            # pattern (see this function's own docstring), where this
386            # heuristic's pairwise-specific asymmetry check doesn't apply.
387            continue
388
389        first_anchored = first_id in anchor_ids
390        second_anchored = second_id in anchor_ids
391        if first_anchored == second_anchored:
392            continue
393
394        anchored_id, unanchored_id = (
395            (first_id, second_id) if first_anchored else (second_id, first_id)
396        )
397        anchored_text = by_id[anchored_id].token if anchored_id in by_id else anchored_id
398        unanchored_text = by_id[unanchored_id].token if unanchored_id in by_id else unanchored_id
399        warnings.append(
400            f"{tok.id} ({tok.token!r}) coordinates {anchored_id} "
401            f"({anchored_text!r}), which anchors its own verbal unit, with "
402            f"{unanchored_id} ({unanchored_text!r}), which does not -- if "
403            "this conjunction is meant to join two verbal expressions "
404            "(rather than a noun/adjective/prepositional-phrase pair), "
405            f"{unanchored_id} is likely missing its own verbalunitid and "
406            "'unit verb'/'root' relation."
407        )
408
409    return warnings

147def tokengraph_to_text(tokengraph: List[TokenAnalysis]) -> str:
148    """Join `tokengraph`'s tokens into one continuous plain-text string,
149    per this module's docstring. Tokens are read in list order (the same
150    order tokengraph_to_mermaid() and validate() assume)."""
151    preposition_ids = _preposition_ids(tokengraph)
152    pieces: List[str] = []
153    previous_class = None
154
155    for tok in tokengraph:
156        if tok.tokentype in IMPLIED_TOKENTYPES:
157            continue
158        cls = _classify(tok, preposition_ids)
159        text = tok.token
160
161        if not pieces:
162            pieces.append(text)
163        elif cls in (_LEFT, _GLUED):
164            pieces.append(text)
165        elif cls == _RIGHT:
166            pieces.append(" " + text)
167        else:  # _NORMAL
168            if previous_class in (_RIGHT, _GLUED):
169                pieces.append(text)
170            else:
171                pieces.append(" " + text)
172
173        previous_class = cls
174
175    return "".join(pieces)

178def tokengraph_to_html(tokengraph: List[TokenAnalysis], *, include_cantillation: bool = True) -> str:
179    """Render `tokengraph` as an HTML string: the same continuous text
180    `tokengraph_to_text()` produces -- identical spacing rules -- except
181    every **lexical** token, and every **proclitic conjunction** carrying a
182    'coordinating conjunction' relation (relationship1 or relationship2),
183    has its text wrapped in a `<span style="...">` colored by the verbal
184    unit it belongs to. Colors come from `verbal_units.assign_verbal_units()`
185    / `assign_verbal_unit_colors()` -- the same assignment and the same
186    first-appearance palette ordering `tokengraph_to_mermaid()` uses for its
187    node coloring -- so a passage rendered here and the same passage's
188    Mermaid diagram color each verbal unit identically.
189
190    The coordinating-conjunction carve-out exists because the proclitic וְ
191    is tokentype "proclitic conjunction", not "lexical", but
192    `assign_verbal_units()` still resolves it to one of the units it
193    coordinates (see that module's docstring). Leaving it unwrapped would
194    visually hide that assignment even though it's a real one.
195
196    Every other non-lexical, non-conjunction token -- paragraph, editorial,
197    maqaf, and a non-conjunction enclitic pronoun -- is still emitted as
198    plain (escaped) text even though `assign_verbal_units()` assigns every
199    token to whichever unit its relations resolve to; this function just
200    doesn't turn that assignment into a span for anything else.
201
202    `include_cantillation` (default `True`, matching arsgrammatica's own
203    tokengraph_to_html(), which has no equivalent exclusion) controls
204    whether cantillation tokens (te'amim -- e.g. the verse-final sof pasuq
205    ׃) are rendered at all. Pass `include_cantillation=False` to omit them
206    from the output entirely -- not just leave them uncolored, the way the
207    other non-lexical tokentypes above are -- for a reading view that
208    foregrounds the lexical/relational content without the accent marks
209    interspersed. This is a deliberate Hebrew-specific divergence from
210    arsgrammatica: Latin punctuation is sparse enough to leave visible by
211    default, but cantillation is dense enough (in principle -- most marks
212    stay embedded in a lexical token's own niqqud under this project's
213    fixtures; see gold_examples.py's own note on that simplification) that
214    a caller may want it gone rather than merely unhighlighted.
215
216    An **implied/elided token** (models.py's IMPLIED_TOKENTYPES) is omitted
217    entirely -- same as tokengraph_to_text() -- rather than rendered with
218    any span: it has no surface text, and unlike
219    `tokengraph_to_mermaid()`'s diagram (which DOES show these), inserting
220    placeholder text into the middle of reconstructed prose here would
221    misrepresent what the passage actually says.
222
223    Every token's text is HTML-escaped (`&`, `<`, `>`, and quote characters)
224    before being emitted, spans or not.
225    """
226    assignment = assign_verbal_units(tokengraph)
227    colors, _warnings = assign_verbal_unit_colors(tokengraph, assignment=assignment)
228    return _tokens_to_html(tokengraph, assignment, colors, include_cantillation=include_cantillation)

306def tokengraph_to_depth_html(
307    tokengraph: List[TokenAnalysis],
308    indent_em: float = _DEFAULT_DEPTH_INDENT_EM,
309    depth: Optional[int] = None,
310) -> Tuple[str, List[str]]:
311    """Render `tokengraph` as HTML illustrating each verbal expression's
312    *depth of subordination* (see verbal_units.compute_subordination_
313    depths()): tokens are assembled sequentially exactly as
314    tokengraph_to_html() does -- same spacing, escaping, and verbal-unit
315    color highlighting -- but grouped into consecutive-run "blocks" by
316    which verbal unit each token belongs to (per assign_verbal_units()),
317    each rendered as its own <div> indented by a CSS margin-left of
318    `depth * indent_em` em -- 0 for an independent clause, 1 for a directly
319    quoted clause it introduces, and so on. All layout is CSS -- no table
320    or nested-list structure is used to produce the indentation.
321
322    `depth`, if given, caps how deep the rendering goes: ONLY blocks whose
323    own depth of subordination is <= `depth` are included in the output.
324    `depth=0` shows root/independent clauses only; omit `depth` (or pass
325    `None`, the default) to show every block. Valid values run from 0 up
326    to verbal_units.max_subordination_depth()'s own return value for this
327    `tokengraph`; a negative `depth` raises ValueError.
328
329    Block boundaries follow assign_verbal_units()'s token-to-unit
330    assignment, with one adjustment: a token whose own tokentype is one of
331    `_GLUED_TOKENTYPES` (enclitic pronoun, proclitic conjunction, maqaf,
332    cantillation) never starts a new block, even when its own assignment
333    differs from the block currently open -- these all attach with no
334    space to a neighboring word (see this module's own docstring), and
335    starting a new block there would split a Hebrew word (or a
336    maqaf-joined pair) across two <div>s. A token with no verbal-unit
337    assignment at all (None) likewise never starts a new block; it folds
338    into whichever block is currently open. Leading tokens before the
339    first resolvable verbal-unit token (rare) default to depth 0.
340
341    A verbal expression whose depth couldn't be resolved (see
342    compute_subordination_depths()) renders at depth 0 rather than
343    raising, with a warning explaining why.
344
345    Returns (html, warnings), combining assign_verbal_unit_colors()'s
346    warnings (colors repeating past 8 verbal units) and
347    compute_subordination_depths()'s (an unresolved governing verbal
348    expression) -- computed the same way, and returned in full, regardless
349    of whether `depth` filters some blocks out of the rendered `html`
350    itself.
351    """
352    if depth is not None and depth < 0:
353        raise ValueError(f"depth must be >= 0 (root clauses only), got {depth!r}")
354
355    assignment = assign_verbal_units(tokengraph)
356    colors, color_warnings = assign_verbal_unit_colors(tokengraph, assignment=assignment)
357    depths, depth_warnings = compute_subordination_depths(tokengraph)
358    warnings = color_warnings + depth_warnings
359
360    blocks = []
361    for tok in tokengraph:
362        unit_id = assignment.get(tok.id)
363        starts_new_block = (
364            unit_id is not None
365            and tok.tokentype not in _GLUED_TOKENTYPES
366            and (not blocks or blocks[-1][0] != unit_id)
367        )
368        if starts_new_block:
369            blocks.append((unit_id, []))
370        elif not blocks:
371            blocks.append((None, []))
372        blocks[-1][1].append(tok)
373
374    lines = []
375    for unit_id, block_tokens in blocks:
376        block_depth = depths.get(unit_id) if unit_id is not None else 0
377        if block_depth is None:
378            block_depth = 0
379        if depth is not None and block_depth > depth:
380            continue
381        block_html = _tokens_to_html(block_tokens, assignment, colors)
382        margin_left = block_depth * indent_em
383        lines.append(
384            f'<div style="margin-left: {margin_left}em; margin-bottom: 0.35em;">'
385            f"{block_html}</div>"
386        )
387
388    return "\n".join(lines), warnings

 47class SyntaxAnalysis(dspy.Signature):
 48    """Analyze the syntax of a passage of Biblical Hebrew according to a
 49    two-part scheme:
 50
 51    (1) a list of verbal expressions. Two constructions count as a verbal
 52        expression: every finite verb, and every participle.
 53
 54        Classify each verbal expression's syntactic type as 'independent'
 55        (main/principal -- syntactically independent, its clause coherent
 56        by itself) or 'direct quote' (occurring in directly quoted speech
 57        introduced by a verb of saying). syntax_model.md documents no other
 58        syntactic_type values in this first draft of the scheme -- in
 59        particular there is no 'dependent' category yet for a subordinate
 60        clause (subordinating conjunctions are listed as "TBA"). A
 61        participle is classified the same way: 'independent' unless it
 62        occurs within quoted speech.
 63
 64        Classify each verbal expression's semantic type too (transitive
 65        active/transitive passive/intransitive/linking verb).
 66
 67    (2) a token-by-token dependency graph. For each token, record up to two
 68        relations to other tokens (by id), using only these relation
 69        labels:
 70
 71        - unit verb (independent): every INDEPENDENT verb (or participle
 72          functioning as one) has relatedtoken1 = the special sentinel
 73          string 'root' -- never an actual token id; no real token may be
 74          assigned the id 'root' -- and relationship1 = 'unit verb'.
 75          Example: in בְּרֵאשִׁית בָּרָא אֱלֹהִים אֵת הַשָּׁמַיִם וְאֵת הָאָרֶץ, the
 76          independent verb בָּרָא has relatedtoken1 = 'root', relationship1 =
 77          'unit verb'.
 78        - direct quote: a verb (or participle) of directly quoted speech
 79          has relatedtoken1 -> the id of the verb of the governing verbal
 80          expression (the verb of saying that introduces/frames the
 81          quotation), relationship1 = 'direct quote' -- matching its own
 82          syntactic_type. Example: in וַיֹּאמֶר אֱלֹהִים יְהִי אֹור וַיְהִי־אֹור,
 83          the verbal unit anchored at יְהִי is direct speech subordinate to
 84          יֹּאמֶר: יְהִי has relatedtoken1 -> יֹּאמֶר's id, relationship1 =
 85          'direct quote'.
 86        - subject / direct object / predicate: a noun or pronoun serving as
 87          the subject of a verbal expression has relatedtoken1 -> the id of
 88          the verb, relationship1 = 'subject'. One functioning as direct
 89          object has relatedtoken1 -> the verb's id, relationship1 =
 90          'direct object'. One functioning as the predicate complement of a
 91          LINKING verb (including an elided-'to be' implied token -- see
 92          (3) below) has relatedtoken1 -> that verb's id, relationship1 =
 93          'predicate'. Example: in Genesis 1.1 (as above), אֱלֹהִים has
 94          relatedtoken1 -> בָּרָא's id, relationship1 = 'subject'; שָׁמַיִם and
 95          אָרֶץ each have relatedtoken1 -> בָּרָא's id, relationship1 = 'direct
 96          object'.
 97        - object marker: the direct object marker אֵת itself has
 98          relatedtoken1 -> the id of the direct object noun/pronoun it
 99          marks, relationship1 = 'object marker'. The marked noun keeps its
100          OWN separate 'direct object' relation to the verb -- this is an
101          additional entry on the marker token, not a replacement for that
102          one. Example: in Genesis 1.1 (as above), the first אֵת has
103          relatedtoken1 -> שָׁמַיִם's id, the second אֵת has relatedtoken1 ->
104          אָרֶץ's id, both relationship1 'object marker'.
105        - coordinating conjunction (single pair): when a coordinating
106          conjunction (the proclitic וְ) joins exactly ONE pair of
107          adjectives, nouns, prepositional phrases, or verbal expressions,
108          it has relatedtoken1 -> the id of the first joined token,
109          relatedtoken2 -> the id of the second, with BOTH relationship1
110          and relationship2 = 'coordinating conjunction' (not an overflow
111          slot here -- this is the one relation that genuinely uses both
112          ends at once). Example: in בְּרֵאשִׁית בָּרָא אֱלֹהִים אֵת הַשָּׁמַיִם
113          וְאֵת הָאָרֶץ, the conjunction וְ (prefixed to אֵת before הָאָרֶץ) has
114          relatedtoken1 -> הַשָּׁמַיִם's id, relatedtoken2 -> הָאָרֶץ's id, both
115          relationship 'coordinating conjunction'.
116        - coordinating conjunction (repeated, as a chain): the proclitic וְ
117          can instead be repeated, prefixed onto EVERY one of a series of
118          two or more coordinated items (most often a chain of narrative
119          wayyiqtol verbs), not just used once between a pair. Annotate
120          this differently from the single-pair case above. Every
121          connector's own relatedtoken1 -> the id of the item it
122          immediately introduces (a real noun, adjective, prepositional
123          phrase, or verbal-expression anchor -- NEVER another connector),
124          relationship1 = 'coordinating conjunction', exactly as in the
125          single-pair case. relatedtoken2 is what differs: the FIRST
126          connector's relatedtoken2 -> the id of the NEXT (second)
127          connector, while every connector AFTER the first has
128          relatedtoken2 -> the id of the PRECEDING connector instead (not
129          the following one); relationship2 = 'coordinating conjunction'
130          for all of them, same as relationship1 -- still not an overflow
131          slot. Each connected item ALSO keeps its own ordinary relation to
132          the rest of the sentence (subject, object of preposition, or
133          whatever fits), completely independent of this chain. Example:
134          in וַיְבָרֶךְ אֱלֹהִים אֶת־יֹום הַשְּׁבִיעִי וַיְקַדֵּשׁ אֹתֹו, two verbal
135          expressions (anchored at בָרֶךְ and קַדֵּשׁ) are coordinated by two
136          instances of וְ: the first וְ (prefixed to יְבָרֶךְ) has
137          relatedtoken1 -> בָרֶךְ's id, relatedtoken2 -> the second וְ's id;
138          the second וְ (prefixed to יְקַדֵּשׁ) has relatedtoken1 -> קַדֵּשׁ's
139          id, relatedtoken2 -> the first וְ's id. בָרֶךְ and קַדֵּשׁ each ALSO
140          have their own relatedtoken1 = 'root', relationship1 = 'unit
141          verb' entries, unaffected by which connector introduces them --
142          being coordinated by וְ does not exempt either verb from its own
143          normal 'unit verb'/'root' entry.
144        - object of preposition: a noun or pronoun functioning as the
145          object of a preposition has relatedtoken1 -> the id of the
146          preposition, relationship1 = 'object of preposition'. Example: in
147          the phrase בְּאֶרֶץ, אֶרֶץ has relatedtoken1 -> בְּ's id,
148          relationship1 = 'object of preposition'.
149        - article: when the article הַ relates to a noun or adjective, it
150          has relatedtoken1 -> the id of that noun or adjective,
151          relationship1 = 'article'. Example: in הַשָּׁמַיִם, the article has
152          relatedtoken1 -> הַשָּׁמַיִם's own lexical-token id, relationship1 =
153          'article'.
154        - construct: when two nouns stand in a construct relation, the
155          governed (related) noun has relatedtoken1 -> the id of the
156          governing noun, relationship1 = 'construct'; the governing noun
157          is separately recorded according to its own function elsewhere
158          in the sentence. Example: in בְּזֵעַת אַפֶּיךָ תֹּאכַל לֶחֶם, אַפֶּי
159          (governed by זֵעַת) has relatedtoken1 -> זֵעַת's id, relationship1 =
160          'construct'; זֵעַת itself is recorded as the object of the
161          preposition בְּ (relatedtoken1 -> בְּ's id, relationship1 = 'object
162          of preposition').
163        - adjectival: an adjective has relatedtoken1 -> the id of the noun
164          it modifies, relationship1 = 'adjectival'. Example: in אֲחִיכֶם
165          הַקָּטֹן, קָּטֹן (modifying אֲחִי) has relatedtoken1 -> אֲחִי's id,
166          relationship1 = 'adjectival'.
167        - adverbial: when a prepositional phrase modifies a verb
168          adverbially, the PREPOSITION ITSELF (not its object) has
169          relatedtoken1 -> the id of the verb, relationship1 = 'adverbial'.
170          The preposition's own object is still separately recorded as
171          'object of preposition', exactly as usual -- this is an
172          additional relation on the preposition, on top of its object's
173          own unaffected relation to it. Example: in בְּרֵאשִׁית בָּרָא אֱלֹהִים
174          אֵת הַשָּׁמַיִם וְאֵת הָאָרֶץ, the preposition בְּ (of the adverbial
175          phrase בְּרֵאשִׁית) has relatedtoken1 -> בָּרָא's id, relationship1 =
176          'adverbial'; its own object רֵאשִׁית has relatedtoken1 -> בְּ's id,
177          relationship1 = 'object of preposition', unchanged.
178
179        Only assign relations described above. Leave relatedtoken/
180        relationship fields unset for tokens with no relation of these
181        kinds -- not every token will have one (syntax_model.md's own "TBA"
182        section names two remaining constructions -- subordinating
183        conjunctions and the relative pronoun אֲשֶׁר -- that have no
184        documented relation at all yet; leave a token unrelated rather than
185        guessing a label for one of these). Use only the token ids given in
186        the input `tokens` list, the sentinel 'root', or a NEW id you
187        create for an implied token (see below), in your output; never
188        invent an id for anything else.
189
190    (3) implied/elided tokens. `diqduq` recognizes one situation where
191        something exists grammatically but has no surface realization in
192        the passage at all: an elided present tense of "to be", which
193        Biblical Hebrew routinely omits from a nominal (verbless) sentence.
194        When this happens, add a NEW entry to `tokengraph` with: a
195        brand-new id, not used by any entry in `tokens` or elsewhere in
196        your own output (see the naming rule below); tokentype 'implied
197        sum'; and no `token` value (leave it unset/None). Also add a
198        matching new entry to `verbalunits`, exactly like any other verbal
199        expression, classified 'independent' (or 'direct quote', if the
200        elided-copula clause is itself directly quoted speech) and
201        'linking verb'. The subject and predicate each relate to this new
202        token exactly as they would to any linking verb ('subject' /
203        'predicate'). Example: in לֹא אֱלֹהִים הֵמָּה ("they are not gods"),
204        the subject is הֵמָּה and the predicate noun is אֱלֹהִים; add a new
205        implied token (tokentype 'implied sum', token=None) anchoring an
206        'independent'/'linking verb' verbal expression, with הֵמָּה related
207        to it as 'subject' and אֱלֹהִים as 'predicate'.
208
209        Naming an implied token's id: append '_implied' to the id of the
210        LAST real token in `tokens` that precedes where the elided "to be"
211        would have stood (or, if the elided word would come before every
212        real token in the sentence, the FIRST real token's id instead). If
213        more than one implied token is ever needed in the same sentence,
214        append '2', '3', ... after '_implied' to keep them unique (e.g.
215        't5_implied', 't5_implied2'). Place the new `tokengraph` entry at
216        the list position where the elided word would have appeared,
217        among the tokens of its own clause.
218    """
219
220    passage: str = dspy.InputField(desc="The Hebrew passage to analyze, exactly as written.")
221    tokens: List[Token] = dspy.InputField(
222        desc="Pre-segmented tokens of the passage, in order, with fixed ids. Reference these ids in your output; do not create new ones."
223    )
224    verbalunits: List[VerbalExpression] = dspy.OutputField(
225        desc="One entry per verbal expression (finite verb or participle) in the passage."
226    )
227    tokengraph: List[TokenAnalysis] = dspy.OutputField(
228        desc=(
229            "One entry per token in `tokens`, in the same order, with its "
230            "type and any relations -- PLUS one additional entry for each "
231            "implied/elided token you add (see this signature's docstring), "
232            "positioned where that token's clause falls in reading order."
233        )
234    )

80def analyze_passage(passage: str, citation: str = "") -> Tuple[List[Sentence], list]:
81    """Convenience wrapper for the common case of a single string rather
82    than a list of citation-labeled CitedText sources -- kept here so
83    callers (diqduq_main.py) have a one-string entry point rather than
84    needing to build a CitedText list themselves for the ordinary case of
85    one passage from one source.
86
87    Wraps `passage` as one CitedText (using `citation` if given, else an
88    empty string -- fine for callers that don't track citations) and runs
89    it through analyze_sources(). Returns (sentences, results) -- one entry
90    per sentence segmentation finds in `passage`, in order (typically one
91    per verse -- see segmentation_dspy.SegmentPassage's docstring).
92    """
93    return analyze_sources([CitedText(citation=citation, text=passage)])

244def validate(tokens: List[Token], result) -> List[str]:
245    """Check that every id the LM produced actually exists among `tokens`
246    -- OR is a legitimately new implied token (tokentype in
247    IMPLIED_TOKENTYPES -- currently just 'implied sum'; see
248    SyntaxAnalysis's docstring) -- and that implied tokens themselves are
249    well-formed. Returns a list of human-readable problem descriptions
250    (empty if clean).
251
252    'root' is a special sentinel value for an independent verb's own
253    relatedtoken1 (see SyntaxAnalysis's docstring) -- it is never treated as
254    an unknown id, but syntax_model.md also requires that no actual token
255    ever be assigned the id 'root', so that's checked here too.
256
257    Implied tokens get their own, narrower checks: a tokengraph entry
258    claiming an IMPLIED_TOKENTYPES value must use a genuinely NEW id (not
259    one already in `tokens`) and must leave `token` unset (None). A
260    non-implied entry, conversely, must use one of `tokens`' own ids and
261    must NOT have `token=None`. This check is purely structural -- it does
262    NOT also require an 'implied sum' token to have a matching
263    `verbalunits` entry; that distinction is documented behavior (see
264    SyntaxAnalysis's docstring), not something this function enforces."""
265    valid_ids = {t.id for t in tokens}
266    problems = []
267
268    if "root" in valid_ids:
269        problems.append(
270            "token id 'root' is reserved as the sentinel relatedtoken1 "
271            "value for independent verbs and must not be assigned to an "
272            "actual token"
273        )
274
275    implied_ids = {tok.id for tok in result.tokengraph if tok.tokentype in IMPLIED_TOKENTYPES}
276    known_ids = valid_ids | implied_ids
277
278    for tok in result.tokengraph:
279        if tok.tokentype in IMPLIED_TOKENTYPES:
280            if tok.id in valid_ids:
281                problems.append(
282                    f"tokengraph entry {tok.id!r} is tokentype={tok.tokentype!r} but "
283                    "reuses an id already in the input `tokens` list -- an "
284                    "implied token must use a new id"
285                )
286            if tok.token is not None:
287                problems.append(
288                    f"tokengraph entry {tok.id!r} is tokentype={tok.tokentype!r} but "
289                    f"has a non-None token value {tok.token!r} -- an implied "
290                    "token's text must be left unset"
291                )
292        else:
293            if tok.id not in valid_ids:
294                problems.append(f"tokengraph entry has unknown id {tok.id!r}")
295            if tok.token is None:
296                allowed = "/".join(repr(t) for t in sorted(IMPLIED_TOKENTYPES))
297                problems.append(
298                    f"tokengraph entry {tok.id!r} has token=None but "
299                    f"tokentype={tok.tokentype!r} -- only {allowed} may "
300                    "omit surface text"
301                )
302        for field in ("relatedtoken1", "relatedtoken2"):
303            val = getattr(tok, field)
304            if val is not None and val != "root" and val not in known_ids:
305                problems.append(f"token {tok.id!r} {field}={val!r} is not a known token id")
306
307    for vu in result.verbalunits:
308        if vu.id not in known_ids:
309            problems.append(f"verbal expression id {vu.id!r} is not a known token id")
310
311    return problems