arsgrammatica

30class Token(BaseModel):
31    """A single pre-segmented token with a stable id.
32 
33    `citation` is optional so this model still works for citation-free
34    callers -- e.g. a test fixture built directly from a canned tokengraph,
35    with no CitedText source at all -- as well as for the citation-aware
36    segmentation stage (segmentation_dspy.py), which is the only thing that
37    actually populates it, knowing which CitedText source unit each token
38    came from."""
39 
40    id: str = Field(description="Stable token id, unique across the whole input, e.g. 't0', 't1', .... As produced directly by segment_sources(), sequential and unique only WITHIN that one call; a caller that combines several such calls (or wants a given passage to keep the same ids across separate runs) should rewrite these first -- see token_ids.assign_passage_scoped_ids().")
41    text: str = Field(description="The token's surface text, exactly as it appears in the source.")
42    citation: Optional[str] = Field(
43        default=None,
44        description="Citation label of the source unit this token came from (e.g. 'Aeneid 1.1'), if known.",
45    )

18class CitedText(BaseModel):
19    """One citable unit of source text -- e.g. one line of poetry, one
20    section of prose -- paired with its citation label. A sequence of
21    these is segmentation_dspy.py's input: sentence boundaries do NOT need
22    to respect CitedText boundaries (one sentence may span several units),
23    but every resulting token still records which unit it came from via
24    Token.citation."""
25 
26    citation: str = Field(description="Citation label for this unit, e.g. 'Aeneid 1.1'.")
27    text: str = Field(description="This unit's raw text, exactly as written.")

48class Sentence(BaseModel):
49    """One sentence's worth of tokens, in reading order, as produced by the
50    LLM-driven segmentation stage (segmentation_dspy.py). Ids are global
51    across whatever ONE segment_sources() call produced this Sentence --
52    numbering continues across sentence boundaries within that call rather
53    than restarting at t0 for each sentence -- so a freshly-segmented
54    Sentence is a contiguous slice of that call's own id sequence, not an
55    independently-numbered unit. A Sentence handed back by
56    pipeline.analyze_sources() (or read back from a file
57    tokenize_ctsdata.py wrote) may instead carry passage-scoped composite
58    ids (token_ids.assign_passage_scoped_ids()) in place of that raw
59    per-call numbering -- see that module's own docstring."""
60 
61    tokens: List[Token] = Field(
62        description="This sentence's tokens, in reading order, using the passage's global token ids."
63    )

 66class VerbalExpression(BaseModel):
 67    """One entry in the table of verbal expressions (syntax_model.md, 'Table
 68    of verbal expressions'). Three constructions count as a verbal
 69    expression: finite verbs, infinitives (when part of indirect speech),
 70    and participles (when they have a *predicate* sense -- e.g. an
 71    ablative-absolute-like "Anco regnante...", 'while Ancus was reigning'
 72    -- rather than a purely *attributive* sense, like an ordinary adjective,
 73    e.g. "consentiens laus", 'universal praise': that case is NOT a verbal
 74    expression at all).
 75 
 76    Each construction has its own set of allowed `syntactic_type` values:
 77    a finite verb is 'independent', 'dependent', 'direct quote' (occurring
 78    in directly quoted speech, e.g. "est" in `"Tuum est," inquit,`), or
 79    'aside' (a verbal expression that interrupts the surrounding syntax,
 80    e.g. "dixerim" in "pace dixerim deum"); an infinitive anchoring an
 81    indirect statement is always 'indirect statement'. syntax_model.md
 82    doesn't specify a dedicated syntactic_type value for participles --
 83    this codebase's convention is to use 'dependent' for a predicate
 84    participle's verbal expression, since a circumstantial participle /
 85    ablative absolute functions like a subordinate clause; flag it if you
 86    intended something else."""
 87 
 88    id: str = Field(
 89        description=(
 90            "The token id (from the input `tokens` list) of the finite verb, "
 91            "infinitive, or predicate-sense participle that anchors this "
 92            "verbal expression. For a compound perfect/pluperfect passive or "
 93            "future-infinitive form (participle + a form of *sum*), use the id "
 94            "of the participle or infinitive itself, NOT the auxiliary form of "
 95            "*sum* (which instead relates to it via the 'auxiliary' relation). "
 96            "For an implied/elided verbal expression (see "
 97            "TokenAnalysis's 'implied sum'/'continued discourse' tokentypes, "
 98            "IMPLIED_TOKENTYPES), use the new implied token's id instead. "
 99            "(A third IMPLIED_TOKENTYPES value, 'implied subject', is NOT "
100            "a verbal expression and never gets an entry here -- see "
101            "TokenAnalysis's own docstring.)"
102        )
103    )
104    syntactic_type: Literal[
105        "independent", "dependent", "direct quote", "aside", "indirect statement"
106    ] = Field(
107        description=(
108            "For a finite verb: 'independent' (main/principal), 'dependent' "
109            "(subordinate/secondary), 'direct quote' (occurring in directly "
110            "quoted speech), or 'aside' (interrupts the surrounding syntax). "
111            "For an infinitive anchoring an indirect statement: 'indirect "
112            "statement'. For a predicate-sense participle: 'dependent' (this "
113            "codebase's convention; syntax_model.md doesn't specify)."
114        )
115    )
116    semantic_type: Literal[
117        "transitive active", "transitive passive", "intransitive", "linking verb"
118    ] = Field(description="The verb's semantic/voice type.")

246class TokenAnalysis(BaseModel):
247    """One entry per token in the dependency graph (syntax_model.md,
248    'Token-level table of dependencies'). Per syntax_model.md's 'Incomplete
249    status' section, not every token will have a relation -- leave the
250    relatedtoken*/relationship* fields unset when none of the documented
251    relations apply.
252
253    Most entries correspond 1:1 to an entry in the input `tokens` list. The
254    exceptions are the three IMPLIED_TOKENTYPES values below, which fall
255    into two DIFFERENT groups. syntax_model.md's 'understood or implied
256    verbal expressions' section documents two situations where a VERBAL
257    EXPRESSION exists grammatically but has no surface realization at all
258    in the passage, and this codebase distinguishes them with two distinct
259    tokentype values rather than one generic 'implied':
260
261    - 'implied sum': an elided form of *sum* -- syntax_model.md's three
262      elided-sum sub-cases (a bare predicate construction, a compound
263      perfect passive/future infinitive missing its auxiliary, or the
264      always-implied present participle of *sum*, which doesn't exist in
265      Latin at all) all use this one value.
266    - 'continued discourse': a governing verb of indirect discourse left
267      unrepeated across several continuation clauses.
268
269    'continued discourse' always anchors its own entry in `verbalunits`,
270    same as any other verbal expression -- and so do two of 'implied
271    sum's own three sub-cases (the bare predicate construction, and the
272    always-implied present participle of *sum*). The THIRD 'implied sum'
273    sub-case -- a compound perfect passive/future infinitive with its
274    auxiliary omitted (e.g. 'facti' for 'facti sunt') -- is the one
275    exception: there, a real, already-present participle/infinitive
276    anchors the verbal expression instead (per VerbalExpression.id's own
277    docstring), and the implied token merely relates to IT via
278    'auxiliary', exactly as a written-out auxiliary would; it gets no
279    `verbalunits` entry of its own in that sub-case. The third
280    IMPLIED_TOKENTYPES value is a DIFFERENT kind of gap entirely -- not a
281    missing verb, but a missing NOUN or pronoun:
282
283    - 'implied subject': a participle's own antecedent (the noun/pronoun
284      it agrees with via 'circumstantial participle') can itself be
285      unexpressed, most often when the participle agrees with a governing
286      verb's own unexpressed subject. This token never anchors a
287      `verbalunits` entry -- it isn't a verbal expression at all -- and
288      instead takes a normal 'subject' relation into the verb whose
289      subject it stands in for, exactly as if that subject had been
290      written out; the participle then relates to THIS token via
291      'circumstantial participle', same as it would to any real noun. See
292      latin_syntax_dspy.SentenceAnalysis's docstring for the worked example.
293
294    For any of the three, add a NEW entry here -- with a NEW id, not
295    present in `tokens` -- rather than skipping the construction entirely;
296    see latin_syntax_dspy.SentenceAnalysis's docstring for the full rules and
297    the id-naming convention."""
298
299    id: str = Field(
300        description=(
301            "For an ordinary entry, must match the id of the corresponding "
302            "entry in the input `tokens` list. For an implied token "
303            "(tokentype in IMPLIED_TOKENTYPES -- 'implied sum', "
304            "'continued discourse', or 'implied subject'), a NEW id not "
305            "used by any entry in `tokens` or elsewhere in this "
306            "tokengraph -- see SentenceAnalysis's docstring for the naming "
307            "convention."
308        )
309    )
310    token: Optional[str] = Field(
311        default=None,
312        description=(
313            "The token's surface text; should match the `text` of the input "
314            "token with this id. Leave as None ONLY for an implied token "
315            "(tokentype 'implied sum', 'continued discourse', or 'implied "
316            "subject') -- one with no surface realization in the passage at "
317            "all; every other tokentype must have real text."
318        ),
319    )
320    tokentype: Literal[
321        "lexical", "enclitic", "punctuation", "numeral", "praenomen", "abbreviation",
322        "implied sum", "continued discourse", "implied subject",
323    ] = Field(
324        description=(
325            "'numeral' is a number written NUMERICALLY -- Roman (e.g. "
326            "'XXV') or Arabic -- rather than spelled out as a word; a "
327            "number spelled out as an ordinary word (e.g. 'decem' for "
328            "'ten') is 'lexical' instead, even though it's semantically a "
329            "number -- e.g. in 'fratres Joseph decem', all three tokens "
330            "are 'lexical', not 'numeral'. "
331            "'praenomen' is specifically an abbreviated Roman first name "
332            "(e.g. 'M.' for Marcus), including its period; 'abbreviation' is "
333            "any other abbreviation, including its period (e.g. 'f.' for "
334            "filius, 'cos.' for consul) -- syntax_model.md's tokenization "
335            "section documents these as two distinct token types, not one. "
336            "'implied sum', 'continued discourse', and 'implied subject' "
337            "each mark a token with NO surface realization at all (see "
338            "this model's own docstring for the distinction) -- the only "
339            "three tokentypes whose `token` field is None and whose `id` "
340            "is not one of the input `tokens`' own ids. 'continued discourse' "
341            "always anchors its own verbal expression (an entry in "
342            "`verbalunits`), and so does 'implied sum' in two of its three "
343            "sub-cases -- EXCEPT the omitted-auxiliary sub-case (e.g. 'facti' "
344            "for 'facti sunt'), where the real, already-present participle "
345            "anchors instead and this implied token only relates to it via "
346            "'auxiliary' (see this model's own docstring). 'implied subject' "
347            "never anchors a verbal expression -- it stands in for an "
348            "unexpressed NOUN or pronoun, not a verb."
349        )
350    )
351 
352    lemma: Optional[str] = Field(default=None, description="Dictionary headword, for lexical tokens. Omit for punctuation.")
353    verbalunitid: Optional[str] = Field(
354        default=None,
355        description="If this token anchors a verbal expression in `verbalunits`, repeat its own id here; otherwise omit.",
356    )
357 
358    relatedtoken1: Optional[str] = Field(
359        default=None,
360        description=(
361            "Token id this token relates to (primary relation). For an "
362            "INDEPENDENT verb's own 'unit verb' relation, use the special "
363            "sentinel string 'root' instead of a token id -- 'root' is "
364            "reserved and must never be assigned as an actual token's id."
365        ),
366    )
367    relationship1: Optional[RelationLabel] = Field(default=None, description="The primary relation type, if any.")
368 
369    relatedtoken2: Optional[str] = Field(default=None, description="Token id this token relates to (secondary relation, used when relation1 is already occupied).")
370    relationship2: Optional[RelationLabel] = Field(default=None, description="The secondary relation type, if any.")

140def tokengraph_to_mermaid(
141    tokengraph: List[TokenAnalysis],
142    orientation: str = "BT",
143    color_by_verbal_unit: bool = True,
144    rank_by_depth: bool = True,
145    aat_depth: Optional[int] = None,
146    show_root: bool = True,
147) -> Tuple[str, List[str]]:
148    """Build a Mermaid `graph` diagram from a tokengraph.
149
150    `orientation` is Mermaid's own flowchart orientation code -- `BT`
151    (bottom-to-top, the default here), `TB`, `LR`, or `RL` -- used verbatim
152    in the diagram's opening line (`graph BT`, `graph LR`, etc.). See
153    https://mermaid.js.org/syntax/flowchart.html for what each value looks
154    like; this function doesn't validate it, so a typo just becomes invalid
155    Mermaid syntax in the output rather than an error here.
156
157    `color_by_verbal_unit` (default True) colors every node by the verbal
158    unit it belongs to, per verbal_units.assign_verbal_units() -- so each
159    clause is visually distinguishable. Verbal units are assigned colors
160    from `_VERBAL_UNIT_PALETTE` in the order their tokens first appear in
161    `tokengraph`; a token assigned to no verbal unit is left with Mermaid's
162    default node styling. The one exception is an implied/elided token
163    (models.py's IMPLIED_TOKENTYPES) -- it always gets its own dedicated
164    `implied` class, colored with `verbal_units._IMPLIED_TOKEN_COLOR`,
165    instead of whatever `_VERBAL_UNIT_PALETTE` color its own verbal unit
166    would otherwise get (see this module's own docstring for why). Pass
167    False to skip coloring and get a plain diagram, as before this
168    parameter existed.
169
170    `rank_by_depth` (default True) makes the diagram's layout respect each
171    verbal expression's own depth in the `aat` package's Agent-Action-
172    Target model (see verbal_units.compute_aat_depths()) -- the same depth
173    aat_bridge.aatgraph() would give that verbal expression's own AAT
174    action node, walking related_node chains to an independent (depth 0)
175    action: every verbal-unit anchor node (any token with `verbalunitid`
176    set to its own id, implied tokens included) at the SAME depth gets
177    chained together with Mermaid's invisible-link syntax (`~~~`), e.g.
178    `t1 ~~~ t6 ~~~ t9` for three anchors all at depth 2. This draws no
179    visible edge and adds no relation of its own -- it only nudges
180    Mermaid's layout engine to keep same-depth verbal expressions level
181    with each other, the same way independent clauses, the dependent
182    clauses one level below them, and so on, end up visually aligned by
183    rank rather than scattered -- and lines this diagram's layout up with
184    an AAT graph of the same sentence (see compute_aat_depths()'s own
185    docstring for exactly how its numbers relate to arsgrammatica's own,
186    richer subordination-depth notion, which this does NOT use). A depth
187    with only one anchor gets no chain at all (nothing to link); unlike
188    the old depth-of-subordination ranking, no anchor is ever excluded for
189    having an unresolved depth -- compute_aat_depths() never leaves one
190    unresolved (see its own docstring for the one malformed-input case,
191    a relation cycle, where that means an arbitrary rather than a
192    meaningful depth, rather than an excluded one). Pass False to skip
193    this and get the diagram's previous, unranked layout.
194
195    `aat_depth`, if given, caps the diagram to nodes belonging to a verbal
196    expression at or within that AAT depth (verbal_units.
197    compute_aat_depths() again -- the SAME notion `rank_by_depth` above
198    uses to align nodes, now used to drop them instead). Every token takes
199    the AAT depth of the verbal unit it belongs to (verbal_units.
200    assign_verbal_units() -- so a whole clause's subject, object, and
201    other ordinary dependents share ONE depth with their own governing
202    verb, exactly the way rendering.tokengraph_to_depth_html()'s own
203    `depth` groups a clause's tokens by verbal_units.compute_
204    subordination_depths()); a token belonging to no verbal unit at all
205    defaults to depth 0 (kept), the same "can't determine, default to root
206    level" fallback used throughout this codebase. This is a DIFFERENT
207    depth notion from dot.py's tokengraph_to_dot(depth=...) filter (a
208    plain graph distance in edges, where each dependent is its own hop
209    deeper than its governor -- see dot.py's own module docstring for the
210    full three-way distinction); `aat_depth=0` shows only root/independent
211    clauses in full (verb plus all of that clause's own dependents), not
212    just root verbs on their own the way tokengraph_to_dot()'s `depth=0`
213    does. `aat_depth=1` adds every verbal expression one AAT-hierarchy
214    level down (and all of ITS clause's own dependents), and so on. A
215    dropped node's own edges (and any kept node's edge that happened to
216    point at it) are skipped, with a warning -- see Returns below -- same
217    "degrade visibly" convention as an edge to punctuation or a missing
218    id. Omit `aat_depth` (or pass `None`, the default) to show every node,
219    same as before this parameter existed; a value at or beyond
220    verbal_units.max_aat_depth()'s own return value for this `tokengraph`
221    shows everything too; a negative `aat_depth` raises `ValueError`.
222    `aat_depth` filtering composes freely with `rank_by_depth` and
223    `color_by_verbal_unit` -- a dropped node is simply never considered by
224    either.
225
226    `show_root` (default True) draws every independent verbal expression's
227    own 'root' relation (relatedtoken1 == 'root', relationship1 == 'unit
228    verb', per syntax_model.md) as a real edge into a single dedicated
229    'root' node, instead of dropping it -- a plain oval (Mermaid's stadium
230    `([...])` shape) labelled "root", deliberately left uncolored/unfilled
231    regardless of `color_by_verbal_unit` (it's never assigned to any
232    verbal unit's classDef/class, so it always keeps Mermaid's own default,
233    unfilled node styling). The node is added only if at least one
234    surviving token actually has a 'root' edge to draw (so an empty or
235    fully-filtered-out diagram never gets an orphan 'root' node), and 'root'
236    itself is exempt from every depth filter above -- an independent verb's
237    anchor is always at depth 0 in every depth notion this codebase uses,
238    so it's never excluded by `aat_depth` on its own account. Pass
239    `show_root=False` to skip the node and its edges entirely, exactly
240    reproducing this function's behavior before this parameter existed.
241
242    Returns (diagram_text, warnings). `warnings` lists any edges that were
243    skipped because they referenced a punctuation token, a node excluded by
244    the `aat_depth` cutoff, or an id not present in `tokengraph` -- worth
245    checking, since a punctuation/missing-id gap usually means the id came
246    from a validation problem upstream (see latin_syntax_dspy.validate) --
247    plus, if `color_by_verbal_unit` is True and the passage has more than 8
248    verbal units, one warning that colors are repeating rather than staying
249    distinct (the palette has 8 slots; see _VERBAL_UNIT_PALETTE).
250    `rank_by_depth` itself never adds a warning (see compute_aat_depths()),
251    and neither does `aat_depth` filtering (same reason: no anchor's AAT
252    depth is ever unresolved).
253    """
254    if aat_depth is not None and aat_depth < 0:
255        raise ValueError(f"aat_depth must be >= 0 (root clauses only), got {aat_depth!r}")
256
257    node_ids = {tok.id for tok in tokengraph if tok.tokentype != "punctuation"}
258
259    if aat_depth is not None:
260        depth_assignment = assign_verbal_units(tokengraph)
261        aat_depths_by_anchor = compute_aat_depths(tokengraph)
262        aat_depth_excluded_ids = {
263            tok.id
264            for tok in tokengraph
265            if tok.id in node_ids
266            and (
267                aat_depths_by_anchor.get(depth_assignment.get(tok.id), 0)
268                if depth_assignment.get(tok.id) is not None
269                else 0
270            )
271            > aat_depth
272        }
273        node_ids -= aat_depth_excluded_ids
274
275    lines = [f"graph {orientation}"]
276    for tok in tokengraph:
277        if tok.id not in node_ids:
278            continue
279        # An implied/elided token (see models.py's IMPLIED_TOKENTYPES) has
280        # no surface text at all -- tok.token is None -- so it needs a
281        # placeholder label rather than crashing _escape_label() on None;
282        # token_label() supplies that. The node's color (below) is what
283        # actually marks it as an implied token, not the label text.
284        label = token_label(tok)
285        # An implied/elided token (models.py's IMPLIED_TOKENTYPES) gets a
286        # rounded-corner rectangle -- Mermaid's `(...)` node shape -- instead
287        # of the plain `[...]` rectangle every other node uses, as a second,
288        # shape-based signal (on top of the dedicated amber color below)
289        # that this node stands in for a word that isn't actually there.
290        open_bracket, close_bracket = (
291            ("(", ")") if tok.tokentype in IMPLIED_TOKENTYPES else ("[", "]")
292        )
293        lines.append(f'    {tok.id}{open_bracket}"{_escape_label(label)}"{close_bracket}')
294
295    # The dedicated 'root' node every independent verb's own 'root' edge
296    # points to (see this module's own docstring) -- added only if at
297    # least one surviving token actually has such an edge to draw, so an
298    # empty or fully-filtered diagram never gets an orphan 'root' node.
299    # Mermaid's stadium shape (`([...])`) for a plain oval look; no
300    # classDef/class is ever applied to it below, so it always keeps
301    # Mermaid's own default, unfilled node styling regardless of
302    # `color_by_verbal_unit`.
303    has_root_edge = show_root and any(
304        tok.relatedtoken1 == "root" for tok in tokengraph if tok.id in node_ids
305    )
306    if has_root_edge:
307        lines.append('    root(["root"])')
308
309    warnings = []
310    for tok in tokengraph:
311        if tok.id not in node_ids:
312            continue
313        for related_field, label_field in (
314            ("relatedtoken1", "relationship1"),
315            ("relatedtoken2", "relationship2"),
316        ):
317            related_id = getattr(tok, related_field)
318            label = getattr(tok, label_field)
319            if related_id is None or label is None:
320                continue
321            if related_id == "root":
322                # An independent verb's own unit-verb relation, per
323                # syntax_model.md -- never a warning-worthy gap either way.
324                # When show_root is True (the default), draw the edge into
325                # the single dedicated 'root' node built above; when False,
326                # skip it silently, exactly as before this parameter
327                # existed.
328                if show_root:
329                    lines.append(f'    {tok.id} -->|{_escape_label(label)}| root')
330                continue
331            if related_id not in node_ids:
332                warnings.append(
333                    f"skipped edge {tok.id} -[{label}]-> {related_id}: "
334                    f"target is punctuation, excluded by the aat_depth "
335                    f"cutoff, or not in tokengraph"
336                )
337                continue
338            lines.append(f'    {tok.id} -->|{_escape_label(label)}| {related_id}')
339
340    if rank_by_depth:
341        # Named aat_depths (not `depths`) to avoid any confusion with the
342        # `aat_depth` FILTERING parameter above -- a different use of the
343        # same underlying compute_aat_depths() numbers (grouping for
344        # ranking here, dropping nodes there), see dot.py's own
345        # analogous rank_by_depth block for the same naming convention.
346        aat_depths = compute_aat_depths(tokengraph)
347
348        # Group every verbal-unit anchor node still in the diagram by its
349        # own AAT-graph depth, preserving tokengraph's own (first-
350        # appearance) order within each group. Unlike the old depth-of-
351        # subordination ranking, compute_aat_depths() never leaves an
352        # anchor's depth unresolved -- aat_depths.get() is None here only
353        # for a non-anchor token (it only ever keys its result by anchor
354        # id), so this `is None` check is purely "is this token an anchor
355        # at all", not "did its depth fail to resolve".
356        depth_groups: dict = {}
357        for tok in tokengraph:
358            if tok.id not in node_ids:
359                continue
360            depth = aat_depths.get(tok.id)
361            if depth is None:
362                continue
363            depth_groups.setdefault(depth, []).append(tok.id)
364
365        rank_lines = [
366            "    " + " ~~~ ".join(ids)
367            for depth in sorted(depth_groups)
368            for ids in (depth_groups[depth],)
369            if len(ids) > 1
370        ]
371        if rank_lines:
372            lines.append("")
373            lines.extend(rank_lines)
374
375    if color_by_verbal_unit:
376        assignment = assign_verbal_units(tokengraph)
377        colors, color_warnings = assign_verbal_unit_colors(tokengraph, assignment=assignment)
378        warnings.extend(color_warnings)
379
380        # Implied tokens (models.py's IMPLIED_TOKENTYPES) always get a
381        # dedicated "caution" amber (_IMPLIED_TOKEN_COLOR) instead of
382        # whatever color their own verbal unit would otherwise get --
383        # regardless of which unit they anchor -- so they're excluded from
384        # every per-unit `member_ids` group below and given their own
385        # classDef/class pair instead. See rendering.py's
386        # tokengraph_to_html() docstring for the matching HTML behavior.
387        implied_ids = [
388            tok.id
389            for tok in tokengraph
390            if tok.id in node_ids and tok.tokentype in IMPLIED_TOKENTYPES
391        ]
392
393        if colors or implied_ids:
394            lines.append("")
395            class_names = {}
396            for i, (unit_id, (fill, stroke, text)) in enumerate(colors.items()):
397                class_name = f"vu{i}"
398                class_names[unit_id] = class_name
399                lines.append(
400                    f"    classDef {class_name} fill:{fill},stroke:{stroke},color:{text};"
401                )
402            for unit_id in colors:
403                member_ids = [
404                    tok.id
405                    for tok in tokengraph
406                    if tok.id in node_ids
407                    and assignment.get(tok.id) == unit_id
408                    and tok.id not in implied_ids
409                ]
410                if member_ids:
411                    lines.append(f"    class {','.join(member_ids)} {class_names[unit_id]};")
412            if implied_ids:
413                fill, stroke, text = _IMPLIED_TOKEN_COLOR
414                lines.append(
415                    f"    classDef implied fill:{fill},stroke:{stroke},color:{text};"
416                )
417                lines.append(f"    class {','.join(implied_ids)} implied;")
418 
419    return "\n".join(lines), warnings

118def token_label(tok: TokenAnalysis) -> str:
119    """The display label for one token: its own surface text (`tok.token`),
120    or -- for an implied/elided token (tokentype in IMPLIED_TOKENTYPES,
121    which has no surface text of its own; `tok.token` is None) -- a
122    placeholder from `_IMPLIED_TOKEN_LABELS` ("elided sum" for "implied
123    sum", "continued discourse" verbatim), falling back to the token's own
124    tokentype string verbatim for any IMPLIED_TOKENTYPES value not listed
125    there (e.g. "implied subject").
126
127    This is the exact label `tokengraph_to_mermaid()` puts in each node's
128    diagram box; it's pulled out as its own function so anything else that
129    needs "the same label the Mermaid diagram uses" for a token --
130    graphs.py's tokengraph_to_networkx(), for one -- can reuse it directly
131    rather than re-deriving (and risking drifting from) the same fallback
132    logic."""
133    return (
134        tok.token
135        if tok.token is not None
136        else _IMPLIED_TOKEN_LABELS.get(tok.tokentype, tok.tokentype)
137    )

422def save_mermaid(
423    tokengraph: List[TokenAnalysis],
424    path: str,
425    orientation: str = "BT",
426    color_by_verbal_unit: bool = True,
427    rank_by_depth: bool = True,
428    aat_depth: Optional[int] = None,
429    show_root: bool = True,
430) -> List[str]:
431    """Write the diagram to `path` (e.g. 'analysis.mmd') and return any
432    warnings from tokengraph_to_mermaid."""
433    diagram, warnings = tokengraph_to_mermaid(
434        tokengraph,
435        orientation=orientation,
436        color_by_verbal_unit=color_by_verbal_unit,
437        rank_by_depth=rank_by_depth,
438        aat_depth=aat_depth,
439        show_root=show_root,
440    )
441    with open(path, "w", encoding="utf-8") as f:
442        f.write(diagram + "\n")
443    return warnings

250def tokengraph_to_dot(
251    tokengraph: List[TokenAnalysis],
252    orientation: str = "BT",
253    color_by_verbal_unit: bool = True,
254    rank_by_depth: bool = True,
255    depth: Optional[int] = None,
256    aat_depth: Optional[int] = None,
257    show_root: bool = True,
258) -> Tuple[str, List[str]]:
259    """Build a Graphviz DOT `digraph` from a tokengraph -- the same
260    diagram tokengraph_to_mermaid() draws (same nodes, same edges, same
261    coloring), as DOT source instead of Mermaid source. See this module's
262    own docstring for why this exists alongside tokengraph_to_mermaid()
263    and what actually rendering the result requires.
264
265    `orientation` maps directly onto DOT's `rankdir` graph attribute --
266    `BT` (bottom-to-top, the default here, matching
267    tokengraph_to_mermaid()'s own default), `TB`, `LR`, or `RL`. Not
268    validated here, same as tokengraph_to_mermaid()'s `orientation` --  a
269    typo just becomes an attribute value Graphviz itself will reject.
270
271    `color_by_verbal_unit` (default True) colors every node by the verbal
272    unit it belongs to, per verbal_units.assign_verbal_units() -- the
273    exact same colors (and the same >8-verbal-units warning) as
274    tokengraph_to_mermaid(), just written as `fillcolor`/`color`/
275    `fontcolor` attributes directly on each node line instead of Mermaid's
276    separate `classDef`/`class` statements (DOT has no equivalent of a
277    named, reusable class -- inline per-node attributes are the idiomatic
278    way to do this). An implied/elided token (IMPLIED_TOKENTYPES) always
279    gets its own dedicated amber (verbal_units._IMPLIED_TOKEN_COLOR)
280    instead of whatever color its own verbal unit would otherwise get,
281    same as tokengraph_to_mermaid(). Pass False for a plain, uncolored
282    diagram.
283
284    `rank_by_depth` (default True) is the reason this module exists
285    alongside tokengraph_to_mermaid() -- see the module docstring. Every
286    verbal-unit anchor node (any token with `verbalunitid` set to its own
287    id, implied tokens included) at the same depth in
288    verbal_units.compute_aat_depths() gets listed together in one
289    `{rank=same; id1; id2; ...}` subgraph statement, which *forces*
290    Graphviz's layout engine to place them on the same rank -- not a nudge,
291    a hard constraint. A depth with only one anchor gets no `rank=same`
292    statement (nothing to align it WITH; unlike Mermaid's `~~~` chain,
293    which genuinely needs 2+ nodes to have anything to link, a
294    single-member `rank=same` would be harmless here too, just an inert
295    statement -- it's skipped for output cleanliness, not necessity). Pass
296    False to skip this and let Graphviz's own layout heuristics place
297    every node.
298
299    `depth`, if given, caps the diagram to nodes at or within that many
300    edges of a root/independent verbal-unit anchor -- compute_graph_depths()
301    above, a plain GRAPH distance along the same relatedtoken1/
302    relatedtoken2 edges drawn as `->` lines below, NOT
303    verbal_units.compute_subordination_depths() (the CLAUSE-level notion
304    behind tokengraph_to_depth_html()'s own indented-HTML `depth` slider --
305    a whole clause's subject, object, and other ordinary dependents share
306    ONE subordination depth with their verb, but each is its own hop of
307    GRAPH depth) and NOT verbal_units.compute_aat_depths() (`rank_by_depth`
308    above). `depth=0` shows ONLY root anchors -- an independent verb with
309    no dependents at all; `depth=1` adds every token one edge away from a
310    root anchor (its subject, object, adverbials, ...); and so on. A token
311    farther than `depth` is dropped entirely: omitted as a node, exactly as
312    if it had never been in `tokengraph`. Omit `depth` (or pass `None`, the
313    default) to show every node, same as before this parameter existed. A
314    `depth` at or beyond max_graph_depth()'s own return value for this
315    `tokengraph` shows everything too; a negative `depth` raises
316    ValueError.
317
318    Dropping a node can leave a KEPT node's edge pointing at a now-excluded
319    one. Such an edge is skipped, with the same combined warning already
320    used for an edge targeting punctuation or a genuinely absent id (see
321    Returns below) -- `depth` filtering degrades visibly rather than
322    emitting a dangling `->` line Graphviz would reject.
323
324    `aat_depth`, if given, is a SEPARATE filter from `depth` above, using
325    the SAME verbal_units.compute_aat_depths() notion `rank_by_depth` uses
326    to align nodes -- now used to drop them instead, mirroring
327    mermaid.tokengraph_to_mermaid()'s own `aat_depth` parameter exactly
328    (see that function's docstring for the full rationale). Every token
329    takes the AAT depth of the verbal unit it belongs to
330    (verbal_units.assign_verbal_units()), so a whole clause's subject,
331    object, and other ordinary dependents share ONE depth with their own
332    governing verb -- unlike `depth` above, where each is its own hop of
333    GRAPH distance. `aat_depth=0` therefore shows a root/independent
334    clause IN FULL (its verb plus every one of that clause's own ordinary
335    dependents), not just the bare root verb the way `depth=0` does. A
336    token belonging to no verbal unit at all defaults to depth 0 (kept),
337    same "can't determine, default to root level" fallback used
338    throughout this codebase. Omit `aat_depth` (or pass `None`, the
339    default) to show every node with respect to this filter; a value at
340    or beyond verbal_units.max_aat_depth()'s own return value for this
341    `tokengraph` shows everything too; a negative `aat_depth` raises
342    `ValueError`. `depth` and `aat_depth` compose freely -- a node is kept
343    only if it survives BOTH cutoffs (whichever are actually given); a
344    dangling edge left by either one is skipped with the same warning.
345
346    `show_root` (default True) draws every independent verbal expression's
347    own 'root' relation (relatedtoken1 == 'root', relationship1 == 'unit
348    verb', per syntax_model.md) as a real edge into a single dedicated
349    'root' node, instead of dropping it -- a plain oval (`shape=oval`)
350    labelled "root", deliberately given no `fillcolor`/`style=filled` (so
351    it's never colored, regardless of `color_by_verbal_unit`) -- mirroring
352    mermaid.tokengraph_to_mermaid()'s own `show_root` parameter exactly.
353    The node is added only if at least one surviving token actually has a
354    'root' edge to draw, and -- like tokengraph_to_mermaid()'s version --
355    'root' is exempt from `depth`/`aat_depth` filtering on its own account:
356    an independent verb's anchor is always depth 0 in both
357    compute_graph_depths() and compute_aat_depths(), so it's never excluded
358    by either cutoff. Pass `show_root=False` to skip the node and its
359    edges entirely, exactly reproducing this function's behavior before
360    this parameter existed.
361
362    Returns `(dot_source, warnings)` -- same shape and same warnings as
363    tokengraph_to_mermaid(): an edge skipped because it targets a
364    punctuation token, a token excluded by the `depth` or `aat_depth`
365    cutoff, or an id not present in `tokengraph` (except the 'root'
366    sentinel, skipped silently, same as there); if `color_by_verbal_unit`
367    is True and the passage has more than 8 verbal units, one warning that
368    colors are repeating. Neither `depth` nor `aat_depth` filtering itself
369    ever adds a warning (compute_graph_depths() and compute_aat_depths()
370    both have no unresolved state -- an unrelated or cyclic token just
371    defaults to depth 0 in either), same as `rank_by_depth`.
372    """
373    if depth is not None and depth < 0:
374        raise ValueError(f"depth must be >= 0 (root nodes only), got {depth!r}")
375    if aat_depth is not None and aat_depth < 0:
376        raise ValueError(f"aat_depth must be >= 0 (root clauses only), got {aat_depth!r}")
377
378    node_ids = {tok.id for tok in tokengraph if tok.tokentype != "punctuation"}
379
380    warnings: List[str] = []
381    if depth is not None:
382        graph_depths = compute_graph_depths(tokengraph)
383        depth_excluded_ids = {tok_id for tok_id, d in graph_depths.items() if d > depth}
384        node_ids -= depth_excluded_ids
385
386    if aat_depth is not None:
387        aat_depth_assignment = assign_verbal_units(tokengraph)
388        aat_depths_by_anchor = compute_aat_depths(tokengraph)
389        aat_depth_excluded_ids = {
390            tok.id
391            for tok in tokengraph
392            if tok.id in node_ids
393            and (
394                aat_depths_by_anchor.get(aat_depth_assignment.get(tok.id), 0)
395                if aat_depth_assignment.get(tok.id) is not None
396                else 0
397            )
398            > aat_depth
399        }
400        node_ids -= aat_depth_excluded_ids
401
402    colors_by_unit = {}
403    implied_ids: set = set()
404    if color_by_verbal_unit:
405        assignment = assign_verbal_units(tokengraph)
406        colors_by_unit, color_warnings = assign_verbal_unit_colors(tokengraph, assignment=assignment)
407        warnings.extend(color_warnings)
408        implied_ids = {
409            tok.id
410            for tok in tokengraph
411            if tok.id in node_ids and tok.tokentype in IMPLIED_TOKENTYPES
412        }
413    else:
414        assignment = {}
415
416    lines = ["digraph tokengraph {", f"    rankdir={orientation};", "    node [shape=box];", ""]
417    for tok in tokengraph:
418        if tok.id not in node_ids:
419            continue
420        color = None
421        if color_by_verbal_unit:
422            if tok.id in implied_ids:
423                color = _IMPLIED_TOKEN_COLOR
424            else:
425                unit_id = assignment.get(tok.id)
426                color = colors_by_unit.get(unit_id) if unit_id is not None else None
427        lines.append(f"    {_dot_id(tok.id)} [{_node_attrs(tok, color)}];")
428
429    # The dedicated 'root' node every independent verb's own 'root' edge
430    # points to (see this module's own docstring) -- added only if at
431    # least one surviving token actually has such an edge to draw, so an
432    # empty or fully-filtered diagram never gets an orphan 'root' node.
433    # `shape=oval` with no `fillcolor`/`style=filled` -- a plain, uncolored
434    # oval regardless of `color_by_verbal_unit`.
435    has_root_edge = show_root and any(
436        tok.relatedtoken1 == "root" for tok in tokengraph if tok.id in node_ids
437    )
438    if has_root_edge:
439        lines.append(f'    {_dot_id("root")} [label="root", shape=oval];')
440
441    lines.append("")
442    for tok in tokengraph:
443        if tok.id not in node_ids:
444            continue
445        for related_field, label_field in (
446            ("relatedtoken1", "relationship1"),
447            ("relatedtoken2", "relationship2"),
448        ):
449            related_id = getattr(tok, related_field)
450            label = getattr(tok, label_field)
451            if related_id is None or label is None:
452                continue
453            if related_id == "root":
454                # An independent verb's own unit-verb relation, per
455                # syntax_model.md -- never a warning-worthy gap either way.
456                # When show_root is True (the default), draw the edge into
457                # the single dedicated 'root' node built above; when False,
458                # skip it silently, exactly as before this parameter
459                # existed.
460                if show_root:
461                    lines.append(f'    {_dot_id(tok.id)} -> {_dot_id("root")} [label="{_escape_label(label)}"];')
462                continue
463            if related_id not in node_ids:
464                warnings.append(
465                    f"skipped edge {tok.id} -[{label}]-> {related_id}: "
466                    f"target is punctuation, excluded by the depth or "
467                    f"aat_depth cutoff, or not in tokengraph"
468                )
469                continue
470            lines.append(f'    {_dot_id(tok.id)} -> {_dot_id(related_id)} [label="{_escape_label(label)}"];')
471
472    if rank_by_depth:
473        aat_depths = compute_aat_depths(tokengraph)
474
475        # Same grouping tokengraph_to_mermaid() builds for its `~~~`
476        # chains -- see that function's own comment for why
477        # aat_depths.get() being None here means "not an anchor", never
478        # "unresolved depth" (compute_aat_depths() has no such state).
479        # This loop's own per-anchor variable is named `rank_depth`, not
480        # `depth` or `aat_depth`, to avoid shadowing EITHER of this
481        # function's own depth-filtering parameters above -- two different
482        # depth notions entirely, see this function's own docstring.
483        depth_groups: dict = {}
484        for tok in tokengraph:
485            if tok.id not in node_ids:
486                continue
487            rank_depth = aat_depths.get(tok.id)
488            if rank_depth is None:
489                continue
490            depth_groups.setdefault(rank_depth, []).append(tok.id)
491
492        rank_lines = [
493            "    {rank=same; " + "; ".join(_dot_id(i) for i in ids) + ";}"
494            for rank_depth in sorted(depth_groups)
495            for ids in (depth_groups[rank_depth],)
496            if len(ids) > 1
497        ]
498        if rank_lines:
499            lines.append("")
500            lines.extend(rank_lines)
501
502    lines.append("}")
503    return "\n".join(lines), warnings

153def compute_graph_depths(tokengraph: List[TokenAnalysis]) -> Dict[str, int]:
154    """Each non-punctuation token's *graph depth*: the number of edges
155    separating it from the nearest root/independent verbal-unit anchor,
156    following the exact same relatedtoken1/relatedtoken2 edges
157    tokengraph_to_dot() itself draws as `->` lines (dependent -> governor)
158    -- the depth notion behind tokengraph_to_dot()'s own `depth` parameter.
159    See this module's own docstring for how this differs from
160    verbal_units.compute_subordination_depths() (rendering.
161    tokengraph_to_depth_html()'s clause-level notion) and
162    verbal_units.compute_aat_depths() (`rank_by_depth`'s notion).
163
164    A root anchor (relatedtoken1 == 'root') is depth 0. Every other
165    token's depth is one more than its PARENT's -- relatedtoken1, falling
166    back to relatedtoken2 only when relatedtoken1 itself doesn't resolve
167    to a usable parent (None, or an id not in `tokengraph`) -- the SAME
168    "relatedtoken1, fall back to relatedtoken2" preference
169    verbal_units.compute_subordination_depths() already uses to chase a
170    verbal expression's own governor. This matters for a token that plays
171    two roles at once, most notably a relative pronoun: e.g. "qui"
172    pointing at its antecedent via relatedtoken1 ('relative pronoun') AND
173    at the dependent verb it's ALSO the subject of via relatedtoken2
174    ('subject') -- that second edge points forward, toward a token that in
175    turn points back at the pronoun itself (its own 'unit verb' relation),
176    a genuine two-way link the data model allows. Taking the shallower of
177    BOTH edges (rather than preferring relatedtoken1) would let that
178    forward edge "cheat" the pronoun's own depth down to whatever the
179    dependent verb's -- itself only computable FROM the pronoun -- happens
180    to resolve to first, collapsing what should be a deeper chain. Only
181    ever falling back to relatedtoken2, never averaging or taking a
182    minimum over both, avoids that: relatedtoken1 alone already resolves
183    to the antecedent here, so relatedtoken2 is simply never consulted for
184    depth (it's still drawn as its own edge below, same as always -- this
185    only affects which relation DEPTH follows).
186
187    A token whose relatedtoken1 AND any fallback relatedtoken2 both fail
188    to resolve (neither set, or pointing at ids not in `tokengraph`), or
189    which is caught in a relation cycle even after preferring
190    relatedtoken1, defaults to depth 0 -- the same "can't determine,
191    default to root level" fallback verbal_units.compute_subordination_
192    depths() and rendering.tokengraph_to_depth_html() both use for their
193    own unresolved cases, rather than raising.
194
195    Returns `{token id: depth}`, one entry per non-punctuation token in
196    `tokengraph` (punctuation is never part of the diagram, so never
197    included here either).
198    """
199    by_id = {tok.id: tok for tok in tokengraph}
200    depths: Dict[str, int] = {}
201    in_progress: set = set()
202
203    def depth_of(tok_id: str) -> int:
204        if tok_id in depths:
205            return depths[tok_id]
206        tok = by_id[tok_id]
207        if tok.relatedtoken1 == "root":
208            depths[tok_id] = 0
209            return 0
210
211        if tok_id in in_progress:
212            # A relation cycle -- fall back to 0 rather than recursing
213            # forever; NOT cached, so a non-cyclic call further up the
214            # stack still computes (and caches) this token's real depth if
215            # some other path reaches it.
216            return 0
217        in_progress.add(tok_id)
218
219        parent_id = None
220        if tok.relatedtoken1 is not None and tok.relatedtoken1 != "root" and tok.relatedtoken1 in by_id:
221            parent_id = tok.relatedtoken1
222        elif tok.relatedtoken2 is not None and tok.relatedtoken2 in by_id:
223            parent_id = tok.relatedtoken2
224
225        result = 1 + depth_of(parent_id) if parent_id is not None else 0
226
227        in_progress.discard(tok_id)
228        depths[tok_id] = result
229        return result
230
231    for tok in tokengraph:
232        if tok.tokentype == "punctuation":
233            continue
234        depth_of(tok.id)
235
236    return depths

239def max_graph_depth(tokengraph: List[TokenAnalysis]) -> Optional[int]:
240    """The highest value compute_graph_depths() assigns to any token in
241    `tokengraph` -- the upper end of the meaningful range for
242    tokengraph_to_dot()'s own `depth` parameter, the same role
243    verbal_units.max_subordination_depth() plays for
244    tokengraph_to_depth_html()'s unrelated depth notion. Returns None for
245    an empty tokengraph, or one with only punctuation."""
246    depths = compute_graph_depths(tokengraph)
247    return max(depths.values()) if depths else None

506def save_dot(
507    tokengraph: List[TokenAnalysis],
508    path: str,
509    orientation: str = "BT",
510    color_by_verbal_unit: bool = True,
511    rank_by_depth: bool = True,
512    depth: Optional[int] = None,
513    aat_depth: Optional[int] = None,
514    show_root: bool = True,
515) -> List[str]:
516    """Write the diagram to `path` (e.g. 'analysis.dot') and return any
517    warnings from tokengraph_to_dot()."""
518    diagram, warnings = tokengraph_to_dot(
519        tokengraph,
520        orientation=orientation,
521        color_by_verbal_unit=color_by_verbal_unit,
522        rank_by_depth=rank_by_depth,
523        depth=depth,
524        aat_depth=aat_depth,
525        show_root=show_root,
526    )
527    with open(path, "w", encoding="utf-8") as f:
528        f.write(diagram + "\n")
529    return warnings

110class GraphMetrics(NamedTuple):
111    """Size/complexity/shape metrics for one tokengraph's NetworkX graph
112    (see tokengraph_to_networkx() and this module's own docstring for the
113    edge orientation these assume -- every "dependent"/"leaf" here is
114    about in-degree, not out-degree).
115
116    Size and complexity:
117
118    - `node_count`/`edge_count`: tokens (excluding punctuation) and
119      relations between them.
120    - `cyclomatic_number`: edges beyond a spanning tree
121      (`edge_count - node_count + weakly_connected_components`) -- 0 for
122      a pure tree; each coordinating conjunction, apposition, or similar
123      construction that gives a token a second governor/dependent beyond
124      strict tree shape adds 1. A direct, single-number answer to "how
125      much non-tree structure does this sentence have".
126    - `is_acyclic`/`longest_chain`: whether the graph is a DAG (it always
127      should be for a well-formed analysis; a cycle means something is
128      malformed, mirroring compute_subordination_depths()'s own
129      cycle-detection warning) and, if so, the length in edges of its
130      longest directed path -- the deepest raw token-to-token embedding
131      chain in the sentence. `longest_chain` is None when a cycle makes
132      it undefined, or the graph is empty.
133
134    Shape:
135
136    - `leaf_count`/`leaf_fraction`: tokens with no dependents of their own
137      (in-degree 0) -- terminal tokens in the dependency structure.
138    - `mean_dependents`/`max_dependents`: the in-degree distribution's
139      mean and max -- how many other tokens point at the typical/busiest
140      token. A sentence with a high max relative to its node count reads
141      as "one token governs almost everything"; a low, even mean/max
142      reads as "shallow and bushy" rather than "deep and chainy".
143    - `relationship_counts`: how many edges carry each relationship label
144      (e.g. `{"subject": 2, "direct object": 1, ...}`) -- what KIND of
145      structure the sentence leans on, not just how much of it there is.
146
147    All fields are 0/0.0/empty (not raised) for an empty graph.
148    """
149
150    node_count: int
151    edge_count: int
152    cyclomatic_number: int
153    is_acyclic: bool
154    longest_chain: Optional[int]
155    leaf_count: int
156    leaf_fraction: float
157    mean_dependents: float
158    max_dependents: int
159    relationship_counts: Dict[str, int]