arsgrammatica

30class Token(BaseModel):
31    """A single pre-segmented token with a stable id.
32 
33    `citation` is optional so this model still works for citation-free
34    callers -- e.g. a test fixture built directly from a canned tokengraph,
35    with no CitedText source at all -- as well as for the citation-aware
36    segmentation stage (segmentation_dspy.py), which is the only thing that
37    actually populates it, knowing which CitedText source unit each token
38    came from."""
39 
40    id: str = Field(description="Stable token id, globally unique and sequential across the whole input, e.g. 't0', 't1', ...")
41    text: str = Field(description="The token's surface text, exactly as it appears in the source.")
42    citation: Optional[str] = Field(
43        default=None,
44        description="Citation label of the source unit this token came from (e.g. 'Aeneid 1.1'), if known.",
45    )

18class CitedText(BaseModel):
19    """One citable unit of source text -- e.g. one line of poetry, one
20    section of prose -- paired with its citation label. A sequence of
21    these is segmentation_dspy.py's input: sentence boundaries do NOT need
22    to respect CitedText boundaries (one sentence may span several units),
23    but every resulting token still records which unit it came from via
24    Token.citation."""
25 
26    citation: str = Field(description="Citation label for this unit, e.g. 'Aeneid 1.1'.")
27    text: str = Field(description="This unit's raw text, exactly as written.")

48class Sentence(BaseModel):
49    """One sentence's worth of tokens, in reading order, as produced by the
50    LLM-driven segmentation stage (segmentation_dspy.py). Token ids are
51    global across the whole passage -- numbering continues across sentence
52    boundaries rather than restarting at t0 for each sentence -- so a
53    Sentence is a contiguous slice of the passage's id sequence, not an
54    independently-numbered unit."""
55 
56    tokens: List[Token] = Field(
57        description="This sentence's tokens, in reading order, using the passage's global token ids."
58    )

 61class VerbalExpression(BaseModel):
 62    """One entry in the table of verbal expressions (syntax_model.md, 'Table
 63    of verbal expressions'). Three constructions count as a verbal
 64    expression: finite verbs, infinitives (when part of indirect speech),
 65    and participles (when they have a *predicate* sense -- e.g. an
 66    ablative-absolute-like "Anco regnante...", 'while Ancus was reigning'
 67    -- rather than a purely *attributive* sense, like an ordinary adjective,
 68    e.g. "consentiens laus", 'universal praise': that case is NOT a verbal
 69    expression at all).
 70 
 71    Each construction has its own set of allowed `syntactic_type` values:
 72    a finite verb is 'independent', 'dependent', 'direct quote' (occurring
 73    in directly quoted speech, e.g. "est" in `"Tuum est," inquit,`), or
 74    'aside' (a verbal expression that interrupts the surrounding syntax,
 75    e.g. "dixerim" in "pace dixerim deum"); an infinitive anchoring an
 76    indirect statement is always 'indirect statement'. syntax_model.md
 77    doesn't specify a dedicated syntactic_type value for participles --
 78    this codebase's convention is to use 'dependent' for a predicate
 79    participle's verbal expression, since a circumstantial participle /
 80    ablative absolute functions like a subordinate clause; flag it if you
 81    intended something else."""
 82 
 83    id: str = Field(
 84        description=(
 85            "The token id (from the input `tokens` list) of the finite verb, "
 86            "infinitive, or predicate-sense participle that anchors this "
 87            "verbal expression. For a compound perfect/pluperfect passive or "
 88            "future-infinitive form (participle + a form of *sum*), use the id "
 89            "of the participle or infinitive itself, NOT the auxiliary form of "
 90            "*sum* (which instead relates to it via the 'auxiliary' relation). "
 91            "For an implied/elided verbal expression (see "
 92            "TokenAnalysis's 'implied sum'/'continued discourse' tokentypes, "
 93            "IMPLIED_TOKENTYPES), use the new implied token's id instead. "
 94            "(A third IMPLIED_TOKENTYPES value, 'implied subject', is NOT "
 95            "a verbal expression and never gets an entry here -- see "
 96            "TokenAnalysis's own docstring.)"
 97        )
 98    )
 99    syntactic_type: Literal[
100        "independent", "dependent", "direct quote", "aside", "indirect statement"
101    ] = Field(
102        description=(
103            "For a finite verb: 'independent' (main/principal), 'dependent' "
104            "(subordinate/secondary), 'direct quote' (occurring in directly "
105            "quoted speech), or 'aside' (interrupts the surrounding syntax). "
106            "For an infinitive anchoring an indirect statement: 'indirect "
107            "statement'. For a predicate-sense participle: 'dependent' (this "
108            "codebase's convention; syntax_model.md doesn't specify)."
109        )
110    )
111    semantic_type: Literal[
112        "transitive active", "transitive passive", "intransitive", "linking verb"
113    ] = Field(description="The verb's semantic/voice type.")

241class TokenAnalysis(BaseModel):
242    """One entry per token in the dependency graph (syntax_model.md,
243    'Token-level table of dependencies'). Per syntax_model.md's 'Incomplete
244    status' section, not every token will have a relation -- leave the
245    relatedtoken*/relationship* fields unset when none of the documented
246    relations apply.
247
248    Most entries correspond 1:1 to an entry in the input `tokens` list. The
249    exceptions are the three IMPLIED_TOKENTYPES values below, which fall
250    into two DIFFERENT groups. syntax_model.md's 'understood or implied
251    verbal expressions' section documents two situations where a VERBAL
252    EXPRESSION exists grammatically but has no surface realization at all
253    in the passage, and this codebase distinguishes them with two distinct
254    tokentype values rather than one generic 'implied':
255
256    - 'implied sum': an elided form of *sum* -- syntax_model.md's three
257      elided-sum sub-cases (a bare predicate construction, a compound
258      perfect passive/future infinitive missing its auxiliary, or the
259      always-implied present participle of *sum*, which doesn't exist in
260      Latin at all) all use this one value.
261    - 'continued discourse': a governing verb of indirect discourse left
262      unrepeated across several continuation clauses.
263
264    'continued discourse' always anchors its own entry in `verbalunits`,
265    same as any other verbal expression -- and so do two of 'implied
266    sum's own three sub-cases (the bare predicate construction, and the
267    always-implied present participle of *sum*). The THIRD 'implied sum'
268    sub-case -- a compound perfect passive/future infinitive with its
269    auxiliary omitted (e.g. 'facti' for 'facti sunt') -- is the one
270    exception: there, a real, already-present participle/infinitive
271    anchors the verbal expression instead (per VerbalExpression.id's own
272    docstring), and the implied token merely relates to IT via
273    'auxiliary', exactly as a written-out auxiliary would; it gets no
274    `verbalunits` entry of its own in that sub-case. The third
275    IMPLIED_TOKENTYPES value is a DIFFERENT kind of gap entirely -- not a
276    missing verb, but a missing NOUN or pronoun:
277
278    - 'implied subject': a participle's own antecedent (the noun/pronoun
279      it agrees with via 'circumstantial participle') can itself be
280      unexpressed, most often when the participle agrees with a governing
281      verb's own unexpressed subject. This token never anchors a
282      `verbalunits` entry -- it isn't a verbal expression at all -- and
283      instead takes a normal 'subject' relation into the verb whose
284      subject it stands in for, exactly as if that subject had been
285      written out; the participle then relates to THIS token via
286      'circumstantial participle', same as it would to any real noun. See
287      latin_syntax_dspy.SentenceAnalysis's docstring for the worked example.
288
289    For any of the three, add a NEW entry here -- with a NEW id, not
290    present in `tokens` -- rather than skipping the construction entirely;
291    see latin_syntax_dspy.SentenceAnalysis's docstring for the full rules and
292    the id-naming convention."""
293
294    id: str = Field(
295        description=(
296            "For an ordinary entry, must match the id of the corresponding "
297            "entry in the input `tokens` list. For an implied token "
298            "(tokentype in IMPLIED_TOKENTYPES -- 'implied sum', "
299            "'continued discourse', or 'implied subject'), a NEW id not "
300            "used by any entry in `tokens` or elsewhere in this "
301            "tokengraph -- see SentenceAnalysis's docstring for the naming "
302            "convention."
303        )
304    )
305    token: Optional[str] = Field(
306        default=None,
307        description=(
308            "The token's surface text; should match the `text` of the input "
309            "token with this id. Leave as None ONLY for an implied token "
310            "(tokentype 'implied sum', 'continued discourse', or 'implied "
311            "subject') -- one with no surface realization in the passage at "
312            "all; every other tokentype must have real text."
313        ),
314    )
315    tokentype: Literal[
316        "lexical", "enclitic", "punctuation", "numeral", "praenomen", "abbreviation",
317        "implied sum", "continued discourse", "implied subject",
318    ] = Field(
319        description=(
320            "'numeral' is a number written NUMERICALLY -- Roman (e.g. "
321            "'XXV') or Arabic -- rather than spelled out as a word; a "
322            "number spelled out as an ordinary word (e.g. 'decem' for "
323            "'ten') is 'lexical' instead, even though it's semantically a "
324            "number -- e.g. in 'fratres Joseph decem', all three tokens "
325            "are 'lexical', not 'numeral'. "
326            "'praenomen' is specifically an abbreviated Roman first name "
327            "(e.g. 'M.' for Marcus), including its period; 'abbreviation' is "
328            "any other abbreviation, including its period (e.g. 'f.' for "
329            "filius, 'cos.' for consul) -- syntax_model.md's tokenization "
330            "section documents these as two distinct token types, not one. "
331            "'implied sum', 'continued discourse', and 'implied subject' "
332            "each mark a token with NO surface realization at all (see "
333            "this model's own docstring for the distinction) -- the only "
334            "three tokentypes whose `token` field is None and whose `id` "
335            "is not one of the input `tokens`' own ids. 'continued discourse' "
336            "always anchors its own verbal expression (an entry in "
337            "`verbalunits`), and so does 'implied sum' in two of its three "
338            "sub-cases -- EXCEPT the omitted-auxiliary sub-case (e.g. 'facti' "
339            "for 'facti sunt'), where the real, already-present participle "
340            "anchors instead and this implied token only relates to it via "
341            "'auxiliary' (see this model's own docstring). 'implied subject' "
342            "never anchors a verbal expression -- it stands in for an "
343            "unexpressed NOUN or pronoun, not a verb."
344        )
345    )
346 
347    lemma: Optional[str] = Field(default=None, description="Dictionary headword, for lexical tokens. Omit for punctuation.")
348    verbalunitid: Optional[str] = Field(
349        default=None,
350        description="If this token anchors a verbal expression in `verbalunits`, repeat its own id here; otherwise omit.",
351    )
352 
353    relatedtoken1: Optional[str] = Field(
354        default=None,
355        description=(
356            "Token id this token relates to (primary relation). For an "
357            "INDEPENDENT verb's own 'unit verb' relation, use the special "
358            "sentinel string 'root' instead of a token id -- 'root' is "
359            "reserved and must never be assigned as an actual token's id."
360        ),
361    )
362    relationship1: Optional[RelationLabel] = Field(default=None, description="The primary relation type, if any.")
363 
364    relatedtoken2: Optional[str] = Field(default=None, description="Token id this token relates to (secondary relation, used when relation1 is already occupied).")
365    relationship2: Optional[RelationLabel] = Field(default=None, description="The secondary relation type, if any.")

131def tokengraph_to_mermaid(
132    tokengraph: List[TokenAnalysis],
133    orientation: str = "BT",
134    color_by_verbal_unit: bool = True,
135    rank_by_depth: bool = True,
136) -> Tuple[str, List[str]]:
137    """Build a Mermaid `graph` diagram from a tokengraph.
138
139    `orientation` is Mermaid's own flowchart orientation code -- `BT`
140    (bottom-to-top, the default here), `TB`, `LR`, or `RL` -- used verbatim
141    in the diagram's opening line (`graph BT`, `graph LR`, etc.). See
142    https://mermaid.js.org/syntax/flowchart.html for what each value looks
143    like; this function doesn't validate it, so a typo just becomes invalid
144    Mermaid syntax in the output rather than an error here.
145
146    `color_by_verbal_unit` (default True) colors every node by the verbal
147    unit it belongs to, per verbal_units.assign_verbal_units() -- so each
148    clause is visually distinguishable. Verbal units are assigned colors
149    from `_VERBAL_UNIT_PALETTE` in the order their tokens first appear in
150    `tokengraph`; a token assigned to no verbal unit is left with Mermaid's
151    default node styling. The one exception is an implied/elided token
152    (models.py's IMPLIED_TOKENTYPES) -- it always gets its own dedicated
153    `implied` class, colored with `verbal_units._IMPLIED_TOKEN_COLOR`,
154    instead of whatever `_VERBAL_UNIT_PALETTE` color its own verbal unit
155    would otherwise get (see this module's own docstring for why). Pass
156    False to skip coloring and get a plain diagram, as before this
157    parameter existed.
158
159    `rank_by_depth` (default True) makes the diagram's layout respect each
160    verbal expression's own depth in the `aat` package's Agent-Action-
161    Target model (see verbal_units.compute_aat_depths()) -- the same depth
162    aat_bridge.attgraph() would give that verbal expression's own AAT
163    action node, walking related_node chains to an independent (depth 0)
164    action: every verbal-unit anchor node (any token with `verbalunitid`
165    set to its own id, implied tokens included) at the SAME depth gets
166    chained together with Mermaid's invisible-link syntax (`~~~`), e.g.
167    `t1 ~~~ t6 ~~~ t9` for three anchors all at depth 2. This draws no
168    visible edge and adds no relation of its own -- it only nudges
169    Mermaid's layout engine to keep same-depth verbal expressions level
170    with each other, the same way independent clauses, the dependent
171    clauses one level below them, and so on, end up visually aligned by
172    rank rather than scattered -- and lines this diagram's layout up with
173    an AAT graph of the same sentence (see compute_aat_depths()'s own
174    docstring for exactly how its numbers relate to arsgrammatica's own,
175    richer subordination-depth notion, which this does NOT use). A depth
176    with only one anchor gets no chain at all (nothing to link); unlike
177    the old depth-of-subordination ranking, no anchor is ever excluded for
178    having an unresolved depth -- compute_aat_depths() never leaves one
179    unresolved (see its own docstring for the one malformed-input case,
180    a relation cycle, where that means an arbitrary rather than a
181    meaningful depth, rather than an excluded one). Pass False to skip
182    this and get the diagram's previous, unranked layout.
183
184    Returns (diagram_text, warnings). `warnings` lists any edges that were
185    skipped because they referenced a punctuation token or an id not present
186    in `tokengraph` -- worth checking, since it usually means the id came
187    from a validation problem upstream (see latin_syntax_dspy.validate) --
188    plus, if `color_by_verbal_unit` is True and the passage has more than 8
189    verbal units, one warning that colors are repeating rather than staying
190    distinct (the palette has 8 slots; see _VERBAL_UNIT_PALETTE).
191    `rank_by_depth` itself never adds a warning -- see compute_aat_depths().
192    """
193    node_ids = {tok.id for tok in tokengraph if tok.tokentype != "punctuation"}
194 
195    lines = [f"graph {orientation}"]
196    for tok in tokengraph:
197        if tok.id not in node_ids:
198            continue
199        # An implied/elided token (see models.py's IMPLIED_TOKENTYPES) has
200        # no surface text at all -- tok.token is None -- so it needs a
201        # placeholder label rather than crashing _escape_label() on None;
202        # token_label() supplies that. The node's color (below) is what
203        # actually marks it as an implied token, not the label text.
204        label = token_label(tok)
205        # An implied/elided token (models.py's IMPLIED_TOKENTYPES) gets a
206        # rounded-corner rectangle -- Mermaid's `(...)` node shape -- instead
207        # of the plain `[...]` rectangle every other node uses, as a second,
208        # shape-based signal (on top of the dedicated amber color below)
209        # that this node stands in for a word that isn't actually there.
210        open_bracket, close_bracket = (
211            ("(", ")") if tok.tokentype in IMPLIED_TOKENTYPES else ("[", "]")
212        )
213        lines.append(f'    {tok.id}{open_bracket}"{_escape_label(label)}"{close_bracket}')
214 
215    warnings = []
216    for tok in tokengraph:
217        if tok.id not in node_ids:
218            continue
219        for related_field, label_field in (
220            ("relatedtoken1", "relationship1"),
221            ("relatedtoken2", "relationship2"),
222        ):
223            related_id = getattr(tok, related_field)
224            label = getattr(tok, label_field)
225            if related_id is None or label is None:
226                continue
227            if related_id == "root":
228                # An independent verb's own unit-verb relation, per
229                # syntax_model.md -- intentionally not a real node, so not
230                # a warning-worthy gap. Just draw no edge for it.
231                continue
232            if related_id not in node_ids:
233                warnings.append(
234                    f"skipped edge {tok.id} -[{label}]-> {related_id}: "
235                    f"target is punctuation or not in tokengraph"
236                )
237                continue
238            lines.append(f'    {tok.id} -->|{_escape_label(label)}| {related_id}')
239
240    if rank_by_depth:
241        depths = compute_aat_depths(tokengraph)
242
243        # Group every verbal-unit anchor node still in the diagram by its
244        # own AAT-graph depth, preserving tokengraph's own (first-
245        # appearance) order within each group. Unlike the old depth-of-
246        # subordination ranking, compute_aat_depths() never leaves an
247        # anchor's depth unresolved -- depths.get() is None here only for a
248        # non-anchor token (it only ever keys its result by anchor id), so
249        # this `is None` check is purely "is this token an anchor at all",
250        # not "did its depth fail to resolve".
251        depth_groups: dict = {}
252        for tok in tokengraph:
253            if tok.id not in node_ids:
254                continue
255            depth = depths.get(tok.id)
256            if depth is None:
257                continue
258            depth_groups.setdefault(depth, []).append(tok.id)
259
260        rank_lines = [
261            "    " + " ~~~ ".join(ids)
262            for depth in sorted(depth_groups)
263            for ids in (depth_groups[depth],)
264            if len(ids) > 1
265        ]
266        if rank_lines:
267            lines.append("")
268            lines.extend(rank_lines)
269
270    if color_by_verbal_unit:
271        assignment = assign_verbal_units(tokengraph)
272        colors, color_warnings = assign_verbal_unit_colors(tokengraph, assignment=assignment)
273        warnings.extend(color_warnings)
274
275        # Implied tokens (models.py's IMPLIED_TOKENTYPES) always get a
276        # dedicated "caution" amber (_IMPLIED_TOKEN_COLOR) instead of
277        # whatever color their own verbal unit would otherwise get --
278        # regardless of which unit they anchor -- so they're excluded from
279        # every per-unit `member_ids` group below and given their own
280        # classDef/class pair instead. See rendering.py's
281        # tokengraph_to_html() docstring for the matching HTML behavior.
282        implied_ids = [
283            tok.id
284            for tok in tokengraph
285            if tok.id in node_ids and tok.tokentype in IMPLIED_TOKENTYPES
286        ]
287
288        if colors or implied_ids:
289            lines.append("")
290            class_names = {}
291            for i, (unit_id, (fill, stroke, text)) in enumerate(colors.items()):
292                class_name = f"vu{i}"
293                class_names[unit_id] = class_name
294                lines.append(
295                    f"    classDef {class_name} fill:{fill},stroke:{stroke},color:{text};"
296                )
297            for unit_id in colors:
298                member_ids = [
299                    tok.id
300                    for tok in tokengraph
301                    if tok.id in node_ids
302                    and assignment.get(tok.id) == unit_id
303                    and tok.id not in implied_ids
304                ]
305                if member_ids:
306                    lines.append(f"    class {','.join(member_ids)} {class_names[unit_id]};")
307            if implied_ids:
308                fill, stroke, text = _IMPLIED_TOKEN_COLOR
309                lines.append(
310                    f"    classDef implied fill:{fill},stroke:{stroke},color:{text};"
311                )
312                lines.append(f"    class {','.join(implied_ids)} implied;")
313 
314    return "\n".join(lines), warnings

109def token_label(tok: TokenAnalysis) -> str:
110    """The display label for one token: its own surface text (`tok.token`),
111    or -- for an implied/elided token (tokentype in IMPLIED_TOKENTYPES,
112    which has no surface text of its own; `tok.token` is None) -- a
113    placeholder from `_IMPLIED_TOKEN_LABELS` ("elided sum" for "implied
114    sum", "continued discourse" verbatim), falling back to the token's own
115    tokentype string verbatim for any IMPLIED_TOKENTYPES value not listed
116    there (e.g. "implied subject").
117
118    This is the exact label `tokengraph_to_mermaid()` puts in each node's
119    diagram box; it's pulled out as its own function so anything else that
120    needs "the same label the Mermaid diagram uses" for a token --
121    graphs.py's tokengraph_to_networkx(), for one -- can reuse it directly
122    rather than re-deriving (and risking drifting from) the same fallback
123    logic."""
124    return (
125        tok.token
126        if tok.token is not None
127        else _IMPLIED_TOKEN_LABELS.get(tok.tokentype, tok.tokentype)
128    )

317def save_mermaid(
318    tokengraph: List[TokenAnalysis],
319    path: str,
320    orientation: str = "BT",
321    color_by_verbal_unit: bool = True,
322    rank_by_depth: bool = True,
323) -> List[str]:
324    """Write the diagram to `path` (e.g. 'analysis.mmd') and return any
325    warnings from tokengraph_to_mermaid."""
326    diagram, warnings = tokengraph_to_mermaid(
327        tokengraph,
328        orientation=orientation,
329        color_by_verbal_unit=color_by_verbal_unit,
330        rank_by_depth=rank_by_depth,
331    )
332    with open(path, "w", encoding="utf-8") as f:
333        f.write(diagram + "\n")
334    return warnings

212def tokengraph_to_dot(
213    tokengraph: List[TokenAnalysis],
214    orientation: str = "BT",
215    color_by_verbal_unit: bool = True,
216    rank_by_depth: bool = True,
217    depth: Optional[int] = None,
218) -> Tuple[str, List[str]]:
219    """Build a Graphviz DOT `digraph` from a tokengraph -- the same
220    diagram tokengraph_to_mermaid() draws (same nodes, same edges, same
221    coloring), as DOT source instead of Mermaid source. See this module's
222    own docstring for why this exists alongside tokengraph_to_mermaid()
223    and what actually rendering the result requires.
224
225    `orientation` maps directly onto DOT's `rankdir` graph attribute --
226    `BT` (bottom-to-top, the default here, matching
227    tokengraph_to_mermaid()'s own default), `TB`, `LR`, or `RL`. Not
228    validated here, same as tokengraph_to_mermaid()'s `orientation` --  a
229    typo just becomes an attribute value Graphviz itself will reject.
230
231    `color_by_verbal_unit` (default True) colors every node by the verbal
232    unit it belongs to, per verbal_units.assign_verbal_units() -- the
233    exact same colors (and the same >8-verbal-units warning) as
234    tokengraph_to_mermaid(), just written as `fillcolor`/`color`/
235    `fontcolor` attributes directly on each node line instead of Mermaid's
236    separate `classDef`/`class` statements (DOT has no equivalent of a
237    named, reusable class -- inline per-node attributes are the idiomatic
238    way to do this). An implied/elided token (IMPLIED_TOKENTYPES) always
239    gets its own dedicated amber (verbal_units._IMPLIED_TOKEN_COLOR)
240    instead of whatever color its own verbal unit would otherwise get,
241    same as tokengraph_to_mermaid(). Pass False for a plain, uncolored
242    diagram.
243
244    `rank_by_depth` (default True) is the reason this module exists
245    alongside tokengraph_to_mermaid() -- see the module docstring. Every
246    verbal-unit anchor node (any token with `verbalunitid` set to its own
247    id, implied tokens included) at the same depth in
248    verbal_units.compute_aat_depths() gets listed together in one
249    `{rank=same; id1; id2; ...}` subgraph statement, which *forces*
250    Graphviz's layout engine to place them on the same rank -- not a nudge,
251    a hard constraint. A depth with only one anchor gets no `rank=same`
252    statement (nothing to align it WITH; unlike Mermaid's `~~~` chain,
253    which genuinely needs 2+ nodes to have anything to link, a
254    single-member `rank=same` would be harmless here too, just an inert
255    statement -- it's skipped for output cleanliness, not necessity). Pass
256    False to skip this and let Graphviz's own layout heuristics place
257    every node.
258
259    `depth`, if given, caps the diagram to nodes at or within that many
260    edges of a root/independent verbal-unit anchor -- compute_graph_depths()
261    above, a plain GRAPH distance along the same relatedtoken1/
262    relatedtoken2 edges drawn as `->` lines below, NOT
263    verbal_units.compute_subordination_depths() (the CLAUSE-level notion
264    behind tokengraph_to_depth_html()'s own indented-HTML `depth` slider --
265    a whole clause's subject, object, and other ordinary dependents share
266    ONE subordination depth with their verb, but each is its own hop of
267    GRAPH depth) and NOT verbal_units.compute_aat_depths() (`rank_by_depth`
268    above). `depth=0` shows ONLY root anchors -- an independent verb with
269    no dependents at all; `depth=1` adds every token one edge away from a
270    root anchor (its subject, object, adverbials, ...); and so on. A token
271    farther than `depth` is dropped entirely: omitted as a node, exactly as
272    if it had never been in `tokengraph`. Omit `depth` (or pass `None`, the
273    default) to show every node, same as before this parameter existed. A
274    `depth` at or beyond max_graph_depth()'s own return value for this
275    `tokengraph` shows everything too; a negative `depth` raises
276    ValueError.
277
278    Dropping a node can leave a KEPT node's edge pointing at a now-excluded
279    one. Such an edge is skipped, with the same combined warning already
280    used for an edge targeting punctuation or a genuinely absent id (see
281    Returns below) -- `depth` filtering degrades visibly rather than
282    emitting a dangling `->` line Graphviz would reject.
283
284    Returns `(dot_source, warnings)` -- same shape and same warnings as
285    tokengraph_to_mermaid(): an edge skipped because it targets a
286    punctuation token, a token excluded by the `depth` cutoff, or an id not
287    present in `tokengraph` (except the 'root' sentinel, skipped silently,
288    same as there); if `color_by_verbal_unit` is True and the passage has
289    more than 8 verbal units, one warning that colors are repeating.
290    `depth` filtering itself never adds a warning (compute_graph_depths()
291    has no unresolved state -- an unrelated or cyclic token just defaults
292    to depth 0), same as `rank_by_depth` -- see compute_aat_depths().
293    """
294    if depth is not None and depth < 0:
295        raise ValueError(f"depth must be >= 0 (root nodes only), got {depth!r}")
296
297    node_ids = {tok.id for tok in tokengraph if tok.tokentype != "punctuation"}
298
299    warnings: List[str] = []
300    if depth is not None:
301        graph_depths = compute_graph_depths(tokengraph)
302        depth_excluded_ids = {tok_id for tok_id, d in graph_depths.items() if d > depth}
303        node_ids -= depth_excluded_ids
304
305    colors_by_unit = {}
306    implied_ids: set = set()
307    if color_by_verbal_unit:
308        assignment = assign_verbal_units(tokengraph)
309        colors_by_unit, color_warnings = assign_verbal_unit_colors(tokengraph, assignment=assignment)
310        warnings.extend(color_warnings)
311        implied_ids = {
312            tok.id
313            for tok in tokengraph
314            if tok.id in node_ids and tok.tokentype in IMPLIED_TOKENTYPES
315        }
316    else:
317        assignment = {}
318
319    lines = ["digraph tokengraph {", f"    rankdir={orientation};", "    node [shape=box];", ""]
320    for tok in tokengraph:
321        if tok.id not in node_ids:
322            continue
323        color = None
324        if color_by_verbal_unit:
325            if tok.id in implied_ids:
326                color = _IMPLIED_TOKEN_COLOR
327            else:
328                unit_id = assignment.get(tok.id)
329                color = colors_by_unit.get(unit_id) if unit_id is not None else None
330        lines.append(f"    {tok.id} [{_node_attrs(tok, color)}];")
331
332    lines.append("")
333    for tok in tokengraph:
334        if tok.id not in node_ids:
335            continue
336        for related_field, label_field in (
337            ("relatedtoken1", "relationship1"),
338            ("relatedtoken2", "relationship2"),
339        ):
340            related_id = getattr(tok, related_field)
341            label = getattr(tok, label_field)
342            if related_id is None or label is None:
343                continue
344            if related_id == "root":
345                # An independent verb's own unit-verb relation, per
346                # syntax_model.md -- intentionally not a real node, so not
347                # a warning-worthy gap. Just draw no edge for it.
348                continue
349            if related_id not in node_ids:
350                warnings.append(
351                    f"skipped edge {tok.id} -[{label}]-> {related_id}: "
352                    f"target is punctuation, excluded by the depth cutoff, "
353                    f"or not in tokengraph"
354                )
355                continue
356            lines.append(f'    {tok.id} -> {related_id} [label="{_escape_label(label)}"];')
357
358    if rank_by_depth:
359        aat_depths = compute_aat_depths(tokengraph)
360
361        # Same grouping tokengraph_to_mermaid() builds for its `~~~`
362        # chains -- see that function's own comment for why
363        # aat_depths.get() being None here means "not an anchor", never
364        # "unresolved depth" (compute_aat_depths() has no such state).
365        # Named aat_depths (not `depths`, and this loop's own variable not
366        # `depth`) to avoid shadowing the `depth` PARAMETER above -- a
367        # different depth notion entirely, see this function's own
368        # docstring.
369        depth_groups: dict = {}
370        for tok in tokengraph:
371            if tok.id not in node_ids:
372                continue
373            aat_depth = aat_depths.get(tok.id)
374            if aat_depth is None:
375                continue
376            depth_groups.setdefault(aat_depth, []).append(tok.id)
377
378        rank_lines = [
379            "    {rank=same; " + "; ".join(ids) + ";}"
380            for aat_depth in sorted(depth_groups)
381            for ids in (depth_groups[aat_depth],)
382            if len(ids) > 1
383        ]
384        if rank_lines:
385            lines.append("")
386            lines.extend(rank_lines)
387
388    lines.append("}")
389    return "\n".join(lines), warnings

115def compute_graph_depths(tokengraph: List[TokenAnalysis]) -> Dict[str, int]:
116    """Each non-punctuation token's *graph depth*: the number of edges
117    separating it from the nearest root/independent verbal-unit anchor,
118    following the exact same relatedtoken1/relatedtoken2 edges
119    tokengraph_to_dot() itself draws as `->` lines (dependent -> governor)
120    -- the depth notion behind tokengraph_to_dot()'s own `depth` parameter.
121    See this module's own docstring for how this differs from
122    verbal_units.compute_subordination_depths() (rendering.
123    tokengraph_to_depth_html()'s clause-level notion) and
124    verbal_units.compute_aat_depths() (`rank_by_depth`'s notion).
125
126    A root anchor (relatedtoken1 == 'root') is depth 0. Every other
127    token's depth is one more than its PARENT's -- relatedtoken1, falling
128    back to relatedtoken2 only when relatedtoken1 itself doesn't resolve
129    to a usable parent (None, or an id not in `tokengraph`) -- the SAME
130    "relatedtoken1, fall back to relatedtoken2" preference
131    verbal_units.compute_subordination_depths() already uses to chase a
132    verbal expression's own governor. This matters for a token that plays
133    two roles at once, most notably a relative pronoun: e.g. "qui"
134    pointing at its antecedent via relatedtoken1 ('relative pronoun') AND
135    at the dependent verb it's ALSO the subject of via relatedtoken2
136    ('subject') -- that second edge points forward, toward a token that in
137    turn points back at the pronoun itself (its own 'unit verb' relation),
138    a genuine two-way link the data model allows. Taking the shallower of
139    BOTH edges (rather than preferring relatedtoken1) would let that
140    forward edge "cheat" the pronoun's own depth down to whatever the
141    dependent verb's -- itself only computable FROM the pronoun -- happens
142    to resolve to first, collapsing what should be a deeper chain. Only
143    ever falling back to relatedtoken2, never averaging or taking a
144    minimum over both, avoids that: relatedtoken1 alone already resolves
145    to the antecedent here, so relatedtoken2 is simply never consulted for
146    depth (it's still drawn as its own edge below, same as always -- this
147    only affects which relation DEPTH follows).
148
149    A token whose relatedtoken1 AND any fallback relatedtoken2 both fail
150    to resolve (neither set, or pointing at ids not in `tokengraph`), or
151    which is caught in a relation cycle even after preferring
152    relatedtoken1, defaults to depth 0 -- the same "can't determine,
153    default to root level" fallback verbal_units.compute_subordination_
154    depths() and rendering.tokengraph_to_depth_html() both use for their
155    own unresolved cases, rather than raising.
156
157    Returns `{token id: depth}`, one entry per non-punctuation token in
158    `tokengraph` (punctuation is never part of the diagram, so never
159    included here either).
160    """
161    by_id = {tok.id: tok for tok in tokengraph}
162    depths: Dict[str, int] = {}
163    in_progress: set = set()
164
165    def depth_of(tok_id: str) -> int:
166        if tok_id in depths:
167            return depths[tok_id]
168        tok = by_id[tok_id]
169        if tok.relatedtoken1 == "root":
170            depths[tok_id] = 0
171            return 0
172
173        if tok_id in in_progress:
174            # A relation cycle -- fall back to 0 rather than recursing
175            # forever; NOT cached, so a non-cyclic call further up the
176            # stack still computes (and caches) this token's real depth if
177            # some other path reaches it.
178            return 0
179        in_progress.add(tok_id)
180
181        parent_id = None
182        if tok.relatedtoken1 is not None and tok.relatedtoken1 != "root" and tok.relatedtoken1 in by_id:
183            parent_id = tok.relatedtoken1
184        elif tok.relatedtoken2 is not None and tok.relatedtoken2 in by_id:
185            parent_id = tok.relatedtoken2
186
187        result = 1 + depth_of(parent_id) if parent_id is not None else 0
188
189        in_progress.discard(tok_id)
190        depths[tok_id] = result
191        return result
192
193    for tok in tokengraph:
194        if tok.tokentype == "punctuation":
195            continue
196        depth_of(tok.id)
197
198    return depths

201def max_graph_depth(tokengraph: List[TokenAnalysis]) -> Optional[int]:
202    """The highest value compute_graph_depths() assigns to any token in
203    `tokengraph` -- the upper end of the meaningful range for
204    tokengraph_to_dot()'s own `depth` parameter, the same role
205    verbal_units.max_subordination_depth() plays for
206    tokengraph_to_depth_html()'s unrelated depth notion. Returns None for
207    an empty tokengraph, or one with only punctuation."""
208    depths = compute_graph_depths(tokengraph)
209    return max(depths.values()) if depths else None

392def save_dot(
393    tokengraph: List[TokenAnalysis],
394    path: str,
395    orientation: str = "BT",
396    color_by_verbal_unit: bool = True,
397    rank_by_depth: bool = True,
398    depth: Optional[int] = None,
399) -> List[str]:
400    """Write the diagram to `path` (e.g. 'analysis.dot') and return any
401    warnings from tokengraph_to_dot()."""
402    diagram, warnings = tokengraph_to_dot(
403        tokengraph,
404        orientation=orientation,
405        color_by_verbal_unit=color_by_verbal_unit,
406        rank_by_depth=rank_by_depth,
407        depth=depth,
408    )
409    with open(path, "w", encoding="utf-8") as f:
410        f.write(diagram + "\n")
411    return warnings

110class GraphMetrics(NamedTuple):
111    """Size/complexity/shape metrics for one tokengraph's NetworkX graph
112    (see tokengraph_to_networkx() and this module's own docstring for the
113    edge orientation these assume -- every "dependent"/"leaf" here is
114    about in-degree, not out-degree).
115
116    Size and complexity:
117
118    - `node_count`/`edge_count`: tokens (excluding punctuation) and
119      relations between them.
120    - `cyclomatic_number`: edges beyond a spanning tree
121      (`edge_count - node_count + weakly_connected_components`) -- 0 for
122      a pure tree; each coordinating conjunction, apposition, or similar
123      construction that gives a token a second governor/dependent beyond
124      strict tree shape adds 1. A direct, single-number answer to "how
125      much non-tree structure does this sentence have".
126    - `is_acyclic`/`longest_chain`: whether the graph is a DAG (it always
127      should be for a well-formed analysis; a cycle means something is
128      malformed, mirroring compute_subordination_depths()'s own
129      cycle-detection warning) and, if so, the length in edges of its
130      longest directed path -- the deepest raw token-to-token embedding
131      chain in the sentence. `longest_chain` is None when a cycle makes
132      it undefined, or the graph is empty.
133
134    Shape:
135
136    - `leaf_count`/`leaf_fraction`: tokens with no dependents of their own
137      (in-degree 0) -- terminal tokens in the dependency structure.
138    - `mean_dependents`/`max_dependents`: the in-degree distribution's
139      mean and max -- how many other tokens point at the typical/busiest
140      token. A sentence with a high max relative to its node count reads
141      as "one token governs almost everything"; a low, even mean/max
142      reads as "shallow and bushy" rather than "deep and chainy".
143    - `relationship_counts`: how many edges carry each relationship label
144      (e.g. `{"subject": 2, "direct object": 1, ...}`) -- what KIND of
145      structure the sentence leans on, not just how much of it there is.
146
147    All fields are 0/0.0/empty (not raised) for an empty graph.
148    """
149
150    node_count: int
151    edge_count: int
152    cyclomatic_number: int
153    is_acyclic: bool
154    longest_chain: Optional[int]
155    leaf_count: int
156    leaf_fraction: float
157    mean_dependents: float
158    max_dependents: int
159    relationship_counts: Dict[str, int]

 51def tokengraph_to_networkx(tokengraph: List[TokenAnalysis]) -> Tuple[nx.MultiDiGraph, List[str]]:
 52    """Build a `networkx.MultiDiGraph` from `tokengraph`, using exactly the
 53    same node/edge selection as tokengraph_to_mermaid() (see this module's
 54    own docstring) -- so a NetworkX graph built here has the same nodes,
 55    the same labels, and the same edges as the diagram drawn for the same
 56    tokengraph, just as a graph object for metric computation rather than
 57    Mermaid source text.
 58
 59    Every node carries two attributes: `label` (via mermaid.token_label(),
 60    the same text/placeholder the diagram shows) and `tokentype` (the
 61    token's own tokentype string, e.g. for a label-aware isomorphism check
 62    later that shouldn't have to look anything up in the original
 63    tokengraph again). Every edge carries one attribute, `relationship`
 64    (the relatedtoken1/relatedtoken2 pair's own relationship1/
 65    relationship2 label).
 66
 67    Returns `(graph, warnings)`; `warnings` mirrors
 68    tokengraph_to_mermaid()'s own list -- a relatedtoken*/relationship*
 69    pair pointing at a punctuation token or an id absent from `tokengraph`
 70    is skipped and reported here, exactly as it is (and isn't drawn)
 71    there; the 'root' sentinel is skipped silently, the same non-issue it
 72    is there, since it was never meant to be a node at all.
 73    """
 74    node_ids = {tok.id for tok in tokengraph if tok.tokentype != "punctuation"}
 75
 76    G: nx.MultiDiGraph = nx.MultiDiGraph()
 77    for tok in tokengraph:
 78        if tok.id not in node_ids:
 79            continue
 80        G.add_node(tok.id, label=token_label(tok), tokentype=tok.tokentype)
 81
 82    warnings: List[str] = []
 83    for tok in tokengraph:
 84        if tok.id not in node_ids:
 85            continue
 86        for related_field, label_field in (
 87            ("relatedtoken1", "relationship1"),
 88            ("relatedtoken2", "relationship2"),
 89        ):
 90            related_id = getattr(tok, related_field)
 91            relationship = getattr(tok, label_field)
 92            if related_id is None or relationship is None:
 93                continue
 94            if related_id == "root":
 95                # An independent verb's own unit-verb relation, per
 96                # syntax_model.md -- intentionally not a real node, so not
 97                # a warning-worthy gap. Just add no edge for it.
 98                continue
 99            if related_id not in node_ids:
100                warnings.append(
101                    f"skipped edge {tok.id} -[{relationship}]-> {related_id}: "
102                    f"target is punctuation or not in tokengraph"
103                )
104                continue
105            G.add_edge(tok.id, related_id, relationship=relationship)
106
107    return G, warnings

162def graph_metrics(G: nx.MultiDiGraph) -> GraphMetrics:
163    """Compute GraphMetrics for `G` (as built by tokengraph_to_networkx(),
164    though this only depends on `G` being a networkx graph with an
165    `relationship` edge attribute -- it doesn't otherwise care how `G` was
166    built). See GraphMetrics's own docstring for what each field means and
167    the in-degree-as-branching convention every "dependent"/"leaf" metric
168    here assumes.
169    """
170    node_count = G.number_of_nodes()
171    edge_count = G.number_of_edges()
172    components = nx.number_weakly_connected_components(G)
173    cyclomatic_number = edge_count - node_count + components
174
175    is_acyclic = nx.is_directed_acyclic_graph(G)
176    longest_chain = nx.dag_longest_path_length(G) if is_acyclic and node_count else None
177
178    in_degrees = [degree for _, degree in G.in_degree()]
179    leaf_count = sum(1 for degree in in_degrees if degree == 0)
180    leaf_fraction = leaf_count / node_count if node_count else 0.0
181    mean_dependents = sum(in_degrees) / node_count if node_count else 0.0
182    max_dependents = max(in_degrees) if in_degrees else 0
183
184    relationship_counts: Dict[str, int] = {}
185    for _, _, relationship in G.edges(data="relationship"):
186        relationship_counts[relationship] = relationship_counts.get(relationship, 0) + 1
187
188    return GraphMetrics(
189        node_count=node_count,
190        edge_count=edge_count,
191        cyclomatic_number=cyclomatic_number,
192        is_acyclic=is_acyclic,
193        longest_chain=longest_chain,
194        leaf_count=leaf_count,
195        leaf_fraction=leaf_fraction,
196        mean_dependents=mean_dependents,
197        max_dependents=max_dependents,
198        relationship_counts=relationship_counts,
199    )

147def assign_verbal_units(tokengraph: List[TokenAnalysis]) -> Dict[str, Optional[str]]:
148    """Return {token id: verbal unit id or None}, one entry per token in
149    `tokengraph` (including punctuation and unrelated tokens, so every id
150    is accounted for -- callers that only care about assigned tokens can
151    filter out the None values themselves).
152 
153    A verbal unit's own anchor token is assigned to itself (its
154    `verbalunitid`). Every other token is assigned to the verbal unit its
155    relations resolve to, per this module's docstring; a token with no
156    resolvable relation (e.g. a bare accusative of place, an enclitic, an
157    emphatic pronoun left unrelated per syntax_model.md's "Incomplete
158    status") gets None.
159
160    A true ablative-absolute noun (its own outgoing relation is "ablative
161    absolute", not some normal clause role) is redirected to the verbal
162    unit of the circumstantial participle it agrees with, rather than to
163    the verb its own relatedtoken1 points at -- see this module's
164    docstring for the full "paucis interiectis diebus ... inscio
165    Collatino ... venit" example. Anything that in turn chains through
166    that noun (an adjective, an appositive) follows it into the
167    participle's unit too, since this redirect happens once, at the noun
168    itself, and every other resolution is unchanged.
169    """
170    by_id = {tok.id: tok for tok in tokengraph}
171
172    # Reverse index: for every token that some OTHER token points at via a
173    # "unit verb" relation, record who points at it. Per syntax_model.md,
174    # a "unit verb" target is always either the literal sentinel 'root'
175    # (from an independent verb -- never a real token) or a subordinating
176    # conjunction/relative pronoun's id (from a dependent verb) -- so a hit
177    # here always means "this token introduces the pointing verb's clause."
178    introduces_clause_for: Dict[str, str] = {}
179    # Reverse index: for every token that some OTHER token points at via a
180    # "circumstantial participle" relation, record who points at it (the
181    # participle -- real or implied -- that agrees with it). Used below to
182    # redirect a TRUE ablative-absolute noun to that participle's own
183    # verbal unit instead of the verb it otherwise points at; a noun a
184    # participle agrees with that fits normally into the clause (its own
185    # outgoing relation isn't "ablative absolute") is left alone and keeps
186    # resolving normally, so this index is consulted but not always used.
187    circumstantial_participle_for: Dict[str, str] = {}
188    for tok in tokengraph:
189        for related_field, label_field in (
190            ("relatedtoken1", "relationship1"),
191            ("relatedtoken2", "relationship2"),
192        ):
193            related = getattr(tok, related_field)
194            label = getattr(tok, label_field)
195            if related is None or related == "root":
196                continue
197            if label == _UNIT_VERB:
198                introduces_clause_for[related] = tok.id
199            elif label == _CIRCUMSTANTIAL_PARTICIPLE:
200                circumstantial_participle_for[related] = tok.id
201
202    resolved: Dict[str, Optional[str]] = {}
203    in_progress: set = set()
204 
205    def resolve(tid: str) -> Optional[str]:
206        if tid in resolved:
207            return resolved[tid]
208        tok = by_id.get(tid)
209        if tok is None:
210            return None
211 
212        if tok.verbalunitid is not None:
213            resolved[tid] = tok.verbalunitid
214            return tok.verbalunitid
215 
216        if tid in in_progress:
217            # A cycle in the relation graph (malformed LM output) -- bail
218            # out on this token rather than recursing forever.
219            return None
220        in_progress.add(tid)
221 
222        result = None
223
224        clause_verb_id = introduces_clause_for.get(tid)
225        if clause_verb_id is not None:
226            result = resolve(clause_verb_id)
227
228        if result is None:
229            participle_id = circumstantial_participle_for.get(tid)
230            is_ablative_absolute = (
231                tok.relationship1 == _ABLATIVE_ABSOLUTE
232                or tok.relationship2 == _ABLATIVE_ABSOLUTE
233            )
234            if participle_id is not None and is_ablative_absolute:
235                result = resolve(participle_id)
236
237        if result is None:
238            for related_field in ("relatedtoken1", "relatedtoken2"):
239                related = getattr(tok, related_field)
240                if related is None or related == "root":
241                    continue
242                result = resolve(related)
243                if result is not None:
244                    break
245 
246        in_progress.discard(tid)
247        resolved[tid] = result
248        return result
249 
250    for tid in by_id:
251        resolve(tid)
252 
253    return resolved

256def assign_verbal_unit_colors(
257    tokengraph: List[TokenAnalysis],
258    assignment: Optional[Dict[str, Optional[str]]] = None,
259) -> Tuple[Dict[str, Tuple[str, str, str]], List[str]]:
260    """Assign each verbal unit found in `tokengraph` a stable (fill, stroke,
261    text) triple from `_VERBAL_UNIT_PALETTE`, using the exact ordering rule
262    `tokengraph_to_mermaid()` uses for its node coloring -- so any other
263    caller wanting "the same colors as the mermaid graph" (currently
264    rendering.py's `tokengraph_to_html()`) gets an identical mapping without
265    re-deriving the rule itself.
266 
267    Order is by first appearance of each verbal unit among tokengraph's
268    *non-punctuation* tokens, since those are the only tokens that become
269    mermaid nodes at all -- a verbal unit whose earliest token happens to be
270    punctuation (it can't be: punctuation tokens aren't assigned to a
271    verbal unit's anchor, but could in principle inherit one from a
272    relation) still gets ordered by its first non-punctuation member.
273 
274    Pass `assignment` (the result of `assign_verbal_units(tokengraph)`) if
275    the caller already computed it, to avoid re-deriving it here; otherwise
276    it's computed internally.
277 
278    Returns `({verbal unit id: (fill, stroke, text)}, warnings)` --
279    `warnings` holds one entry, with the same wording
280    `tokengraph_to_mermaid()` uses, if there are more distinct verbal units
281    than palette slots (colors repeat past the 8th unit). A verbal unit id
282    absent from the returned dict was never assigned to any non-punctuation
283    token -- callers should treat that the same as "no verbal unit" (no
284    coloring), same as `tokengraph_to_mermaid()` does.
285    """
286    if assignment is None:
287        assignment = assign_verbal_units(tokengraph)
288 
289    non_punctuation_ids = {tok.id for tok in tokengraph if tok.tokentype != "punctuation"}
290 
291    unit_order: List[str] = []
292    seen_units = set()
293    for tok in tokengraph:
294        if tok.id not in non_punctuation_ids:
295            continue
296        unit_id = assignment.get(tok.id)
297        if unit_id is not None and unit_id not in seen_units:
298            seen_units.add(unit_id)
299            unit_order.append(unit_id)
300 
301    warnings: List[str] = []
302    if len(unit_order) > len(_VERBAL_UNIT_PALETTE):
303        warnings.append(
304            f"{len(unit_order)} verbal units but only {len(_VERBAL_UNIT_PALETTE)} "
305            "distinct colors -- colors repeat and may be ambiguous between units"
306        )
307 
308    colors = {
309        unit_id: _VERBAL_UNIT_PALETTE[i % len(_VERBAL_UNIT_PALETTE)]
310        for i, unit_id in enumerate(unit_order)
311    }
312    return colors, warnings