{ "pack_id": "persona-panel-second-opinion", "title": "Persona panel and second opinion", "version": "0.1.0", "summary": "A collection of nouns and verbs for independent second opinions before a decision: a panel of reviewers, each with a stated perspective and, where possible, a model family different from the producer's; takes given alone from the same sealed packet, with no reviewer seeing another's; an independence check before any view is read; a synthesis that leads with where the reviewers disagree and routes each disagreement, never by a vote; an audit by someone who wrote none of it; and a person who decides. It reuses the taste palettes and evaluation design packs where they overlap.", "dependencies": [ "taste-palettes", "evaluation-design", "policy-and-law-watch" ], "boundaries": [ "Each reviewer works alone from the same packet; no reviewer sees another's view before all are sealed.", "Perspective cards are roles, never real people; no one is imitated or claimed to be represented.", "Reviewers spend only their workers' own allowances; this pack grants no spending and calls no model itself.", "The synthesis decides nothing; only a person decides.", "Workers create nouns and propose verbs; an authorized requester enqueues the proposals worth doing.", "Importing the pack writes nothing and enqueues nothing." ], "nouns": [ { "slug": "pack", "category": "noun.skill_pack", "role": "pack", "title": "A collection of nouns and verbs for independent second opinions from a panel of stated perspectives", "value": { "lets_workers": "Get second opinions before a decision: define a panel of independent reviewers, each with a stated perspective and, where possible, a model family different from the producer's; have each give a take alone from the same sealed packet, with no reviewer seeing another's; check that independence before anyone reads the views; write a synthesis that leads with where the reviewers disagree and where each disagreement goes next; have someone else audit it; and leave the decision to a person.", "not_for": "Choosing between finished options by taste, which the taste palettes pack does, or measuring which option is better, which the evaluation design pack does. A panel routes such disagreements to them.", "flow": [ { "verb": { "$verb": "persona-panel-second-opinion.define-panel" }, "when": "The plan or draft, the decision it informs and who produced it." }, { "verb": { "$verb": "persona-panel-second-opinion.give-view" }, "when": "Each reviewer alone, as a separate attempt." }, { "verb": { "$verb": "persona-panel-second-opinion.check-independence" }, "when": "Before anyone reads a view." }, { "verb": { "$verb": "persona-panel-second-opinion.synthesize-disagreements" }, "when": "The sealed views, read in the fixed order." }, { "verb": { "$verb": "persona-panel-second-opinion.audit-synthesis" }, "when": "Someone who wrote none of it checks the quotes and routes." }, { "verb": { "$verb": "persona-panel-second-opinion.decide" }, "when": "A person decides, or decides later." }, { "verb": { "$verb": "evaluation-design.design-evaluation" }, "when": "A factual disagreement gets an evaluation." }, { "verb": { "$verb": "taste-palettes.draft-embodiments" }, "when": "A disagreement of taste becomes a palette." } ], "guardrails": [ { "$pin": "guardrail.independent-takes" }, { "$pin": "guardrail.disagreement-first" }, { "$pin": "guardrail.different-family-where-possible" }, { "$pin": "policy-and-law-watch:guardrail.person-decides" } ], "perspective_cards": [ { "$pin": "perspective.served-person" }, { "$pin": "perspective.skeptic" }, { "$pin": "perspective.operator" }, { "$pin": "perspective.steward" }, { "$pin": "perspective.newcomer" }, { "$pin": "perspective.safety-and-privacy" } ], "worked_example": "Synthetic: a literacy program's plan, drafted by a model run, to replace paper sign-in sheets with tablet check-in. Three reviewers on three families, one shared with the producer and labeled, take it alone. Their sealed views disagree on a fact, whether check-ins survive a site's evening wifi outages, and on a value, whether quicker check-in is worth a screen some learners cannot read. The program director proceeds with changes and leaves the factual question to a week-long test." }, "relations": [ [ "requires_noun", "method" ], [ "requires_noun", "guardrail.independent-takes" ], [ "requires_noun", "guardrail.disagreement-first" ], [ "requires_noun", "guardrail.different-family-where-possible" ], [ "requires_noun", "rules.panel-composition" ], [ "requires_noun", "rules.independence" ], [ "requires_noun", "rules.synthesis" ], [ "requires_noun", "template.synthesis" ], [ "requires_noun", "taste-palettes:method" ], [ "requires_noun", "evaluation-design:method" ] ] }, { "slug": "method", "category": "noun.skill_pack", "role": "method", "title": "How a panel gives independent takes and a person decides", "category_reason": "A reusable procedure with guardrails, followed by every verb in this pack.", "value": { "lifecycle": [ "define-panel records the producer and its model family, picks three to five perspective cards, gives each reviewer a family, and fixes the shared packet, the deadline and the reading seed.", "give-view runs once per reviewer as a separate attempt, each sealed on submission.", "check-independence confirms every view used the packet, was sealed in time and names no other reviewer, before anyone reads them.", "synthesize-disagreements reads the views in the fixed order and leads with where they disagree.", "audit-synthesis, by someone who wrote none of it, rechecks every quote and route.", "decide is a person's: proceed, proceed with changes, do not proceed or decide later." ], "rules": [ "Independence is the point: a view written after reading another is a second copy, not a second opinion.", "Disagreements are the output; agreement among reviewers who share blind spots is weak evidence.", "A factual disagreement is settled by evidence, a value disagreement by the person, a taste disagreement by a palette and a scope disagreement by asking the requester.", "Each reviewer spends only its worker's own allowance; this pack grants no spending.", "Workers create nouns and propose verbs; they never enqueue." ], "follow_ups": { "factual": { "$verb": "evaluation-design.design-evaluation" }, "taste": { "$verb": "taste-palettes.draft-embodiments" }, "choice": { "$verb": "taste-palettes.record-choice" }, "process_problem": { "$verb": "findings-to-fixes.record-finding" } } } }, { "slug": "guardrail.independent-takes", "category": "noun.skill_pack", "role": "guardrail", "title": "Guardrail: every take is independent", "category_reason": "A guardrail is a reusable rule every verb in this work follows, with the check that proves it.", "value": { "rule": "Each reviewer works alone from the same sealed packet: the artifact, the question and its own perspective card. No reviewer sees another's take, the producer's view of its own work or a draft synthesis until every take is sealed.", "how_it_is_checked": "Every view names the packet digest it read and its seal time, and the independence check finds no view naming another reviewer and none read before all were sealed." } }, { "slug": "guardrail.disagreement-first", "category": "noun.skill_pack", "role": "guardrail", "title": "Guardrail: the synthesis leads with disagreement", "category_reason": "A guardrail is a reusable rule every verb in this work follows, with the check that proves it.", "value": { "rule": "A synthesis leads with where reviewers disagree, quoting each side from the sealed views, and keeps a point only one reviewer raised at full weight. Agreement gets a short list, and a head count never settles a disagreement.", "how_it_is_checked": "Every disagreement quotes each position from its view, and the summary holds no majority verdict." } }, { "slug": "guardrail.different-family-where-possible", "category": "noun.skill_pack", "role": "guardrail", "title": "Guardrail: reviewers on other model families where possible", "category_reason": "A guardrail is a reusable rule every verb in this work follows, with the check that proves it.", "value": { "rule": "Reviewers run on a model family other than the producer's wherever one is available, and on more than one family across the panel. A reviewer on the producer's family is named with the reason, and the synthesis calls that part of the panel correlated.", "how_it_is_checked": "Every reviewer's family is recorded beside the producer's, and every same-family reviewer has a stated reason." } }, { "slug": "rules.panel-composition", "category": "noun.reflex_procedure", "role": "rules", "title": "Who sits on a panel", "category_reason": "A zero-call deterministic procedure: the same request and allowance give the same panel shape.", "value": { "rules": [ "Three to five reviewers, each with a different perspective card; at least one looks for how the plan fails and one stands for the people it serves.", "A perspective is a role, never a real person and never an imitation of one.", "The producer, whether a person or a model run, is not on its own panel, and neither is the synthesizer.", "Record the producer's model family and each reviewer's. Prefer families other than the producer's, and more than one across the panel; a reviewer on the producer's family needs a stated reason.", "Reviewer ids are opaque, such as r1 and r2, so an id says nothing about a perspective or a family.", "Each reviewer is a separate attempt with its own lease and an execution report naming the exact provider and model." ], "size": { "min": 3, "max": 5 }, "model_calls": 0 } }, { "slug": "rules.independence", "category": "noun.reflex_procedure", "role": "rules", "title": "Keeping the takes independent", "category_reason": "A zero-call deterministic procedure: the same views and seed give the same check and order.", "value": { "rules": [ "Every reviewer gets the same packet: the artifact by digest, the question and the instructions; only the perspective card differs.", "The packet leaves out the producer's view of its own work and every other reviewer's take.", "Reviewers share no conversation, memory, cache or scratch space.", "A view is sealed by the SHA-256 of its canonical JSON when submitted, and nobody reads any view until all are sealed or the deadline passes.", "A late view is kept and marked late. A view that names or quotes another reviewer is excluded and redone, because its independence is lost.", "Views are read in ascending order of the SHA-256 of the seed, a colon and the reviewer id, so their order does not depend on who finished first." ], "model_calls": 0 }, "relations": [ [ "derives_from", "taste-palettes:method", "A recorded shuffle, so position does not decide." ] ] }, { "slug": "rules.synthesis", "category": "noun.reflex_procedure", "role": "rules", "title": "Synthesizing where reviewers disagree", "category_reason": "A zero-call deterministic procedure: the same views give the same structure.", "value": { "rules": [ "Lead with disagreements. Quote every side exactly from the sealed views, with the reviewer id and perspective.", "Name each disagreement's kind and route, and say what would resolve it.", "Keep agreements to a short list, and list every point only one reviewer raised.", "Never settle a disagreement by counting reviewers; a count is information only, with the panel size.", "Say plainly when there is no disagreement, and why it may be real or correlated.", "The synthesis recommends nothing; a person decides." ], "kinds": { "factual": "Evidence or a measurement could settle it: route to an evaluation.", "value": "It turns on what matters more: route to the person who decides.", "taste": "It turns on preference between versions: route to a taste palette.", "scope": "Reviewers assumed different goals or limits: route to the requester." }, "routes": { "factual": "evaluation", "value": "person", "taste": "taste_palette", "scope": "requester" }, "model_calls": 0 }, "relations": [ [ "requires_noun", "evaluation-design:method" ] ] }, { "slug": "perspective.served-person", "category": "noun.prompt_shim", "role": "perspective_card", "title": "Perspective card: the person it serves", "category_reason": "A short overlay a reviewer works under, added to the shared packet; it carries no data.", "value": { "perspective": "the person it serves", "stands_for": "Someone the decision is meant to serve, meeting it at its hardest moment.", "cares_about": [ "dignity", "time and effort asked of them", "being understood", "not being left out" ], "asks": [ "What does this ask of me, and what happens if I cannot do it?" ], "blind_spots": "Costs and operations behind the scenes.", "say_too": "What would change your mind, and how sure you are of each point.", "never": "Claim to speak for real people or imitate anyone; this is a lens, not a survey." } }, { "slug": "perspective.skeptic", "category": "noun.prompt_shim", "role": "perspective_card", "title": "Perspective card: the skeptic", "category_reason": "A short overlay a reviewer works under, added to the shared packet; it carries no data.", "value": { "perspective": "the skeptic", "stands_for": "Someone looking for how the plan fails, on its first bad day.", "cares_about": [ "failure modes", "assumptions nobody checked", "what happens when something is down" ], "asks": [ "What has to be true for this to work, and what happens when it is not?" ], "blind_spots": "Benefits that are real but quiet.", "say_too": "What would change your mind, and how sure you are of each point.", "never": "Claim to speak for real people or imitate anyone; this is a lens, not a survey." } }, { "slug": "perspective.operator", "category": "noun.prompt_shim", "role": "perspective_card", "title": "Perspective card: the operator", "category_reason": "A short overlay a reviewer works under, added to the shared packet; it carries no data.", "value": { "perspective": "the operator", "stands_for": "The person who runs it day to day and fixes it when it breaks.", "cares_about": [ "workload", "handoffs", "what breaks at the worst time", "who fixes it" ], "asks": [ "Who does this on a busy evening, and how long does it take?" ], "blind_spots": "How it feels to the people served.", "say_too": "What would change your mind, and how sure you are of each point.", "never": "Claim to speak for real people or imitate anyone; this is a lens, not a survey." } }, { "slug": "perspective.steward", "category": "noun.prompt_shim", "role": "perspective_card", "title": "Perspective card: the steward", "category_reason": "A short overlay a reviewer works under, added to the shared packet; it carries no data.", "value": { "perspective": "the steward", "stands_for": "The person accountable for money, time and promises to funders.", "cares_about": [ "cost now and over a year", "commitments already made", "what must be reported" ], "asks": [ "What does this cost over a year, and what did we promise?" ], "blind_spots": "Effects that never show up in a budget.", "say_too": "What would change your mind, and how sure you are of each point.", "never": "Claim to speak for real people or imitate anyone; this is a lens, not a survey." } }, { "slug": "perspective.newcomer", "category": "noun.prompt_shim", "role": "perspective_card", "title": "Perspective card: the newcomer", "category_reason": "A short overlay a reviewer works under, added to the shared packet; it carries no data.", "value": { "perspective": "the newcomer", "stands_for": "Someone meeting it for the first time, with no one to ask.", "cares_about": [ "clear first steps", "plain words", "what to do when stuck" ], "asks": [ "Could I use this on my first day without help?" ], "blind_spots": "Why the experienced accept its complexity.", "say_too": "What would change your mind, and how sure you are of each point.", "never": "Claim to speak for real people or imitate anyone; this is a lens, not a survey." } }, { "slug": "perspective.safety-and-privacy", "category": "noun.prompt_shim", "role": "perspective_card", "title": "Perspective card: safety and privacy", "category_reason": "A short overlay a reviewer works under, added to the shared packet; it carries no data.", "value": { "perspective": "safety and privacy", "stands_for": "Someone guarding people's safety and the information kept about them.", "cares_about": [ "what data is kept and for how long", "who can see it", "who could be hurt if it leaked" ], "asks": [ "What happens to people's information, and who could be hurt by this?" ], "blind_spots": "Convenience people value.", "say_too": "What would change your mind, and how sure you are of each point.", "never": "Claim to speak for real people or imitate anyone; this is a lens, not a survey." } }, { "slug": "template.panel", "category": "noun.seed_hypothesis", "role": "template", "title": "Template: a panel: the question, the artifact by digest, the producer and its family, the reviewers with their perspectives and families, the packet digest, the deadline and the reading seed", "category_reason": "A panel states who should see the artifact differently, which the views then test.", "value": { "describes": "a panel: the question, the artifact by digest, the producer and its family, the reviewers with their perspectives and families, the packet digest, the deadline and the reading seed", "schema": { "type": "object", "additionalProperties": false, "required": [ "panel_id", "question", "artifact", "producer", "reviewers", "packet_sha256", "deadline", "shuffle_seed", "decision_owner_role" ], "properties": { "panel_id": { "type": "string", "minLength": 1, "maxLength": 40 }, "question": { "type": "string", "minLength": 1, "maxLength": 400 }, "artifact": { "type": "object", "additionalProperties": false, "required": [ "ref", "sha256" ], "properties": { "ref": { "type": "string", "minLength": 1, "maxLength": 300 }, "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" } } }, "producer": { "type": "object", "additionalProperties": false, "required": [ "kind", "model_family", "on_panel" ], "properties": { "kind": { "type": "string", "enum": [ "person", "model_run" ] }, "model_family": { "type": "string", "minLength": 1, "maxLength": 60 }, "on_panel": { "type": "boolean" } } }, "reviewers": { "type": "array", "minItems": 3, "maxItems": 7, "items": { "type": "object", "additionalProperties": false, "required": [ "reviewer_id", "perspective", "card_ref", "model_family", "same_family_reason" ], "properties": { "reviewer_id": { "type": "string", "pattern": "^[a-z][a-z0-9-]{0,19}$", "maxLength": 20 }, "perspective": { "type": "string", "minLength": 1, "maxLength": 80 }, "card_ref": { "type": "string", "minLength": 1, "maxLength": 300 }, "model_family": { "type": "string", "minLength": 1, "maxLength": 60 }, "same_family_reason": { "type": "string", "minLength": 0, "maxLength": 300 } } } }, "packet_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, "deadline": { "type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}(:[0-9]{2})?Z$", "maxLength": 30 }, "shuffle_seed": { "type": "integer", "minimum": 0, "maximum": 2147483647 }, "decision_owner_role": { "type": "string", "minLength": 1, "maxLength": 80 } } }, "rules": [ "same_family_reason is empty unless the reviewer shares the producer's family." ] } }, { "slug": "template.view", "category": "noun.eval_run", "role": "template", "title": "Template: one reviewer's independent take: the packet it read, its stance, its points with confidence, what would change its mind and when it was sealed", "category_reason": "A view is one recorded judgment run.", "value": { "describes": "one reviewer's independent take: the packet it read, its stance, its points with confidence, what would change its mind and when it was sealed", "schema": { "type": "object", "additionalProperties": false, "required": [ "panel_id", "reviewer_id", "perspective", "model_family", "packet_sha256", "saw_other_views", "stance", "summary", "points", "would_change_my_mind", "sealed_at" ], "properties": { "panel_id": { "type": "string", "minLength": 1, "maxLength": 40 }, "reviewer_id": { "type": "string", "pattern": "^[a-z][a-z0-9-]{0,19}$", "maxLength": 20 }, "perspective": { "type": "string", "minLength": 1, "maxLength": 80 }, "model_family": { "type": "string", "minLength": 1, "maxLength": 60 }, "packet_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, "saw_other_views": { "type": "boolean" }, "stance": { "type": "string", "enum": [ "support", "support_with_changes", "oppose", "unsure" ] }, "summary": { "type": "string", "minLength": 1, "maxLength": 600 }, "points": { "type": "array", "minItems": 1, "maxItems": 10, "items": { "type": "object", "additionalProperties": false, "required": [ "id", "kind", "text", "why", "confidence" ], "properties": { "id": { "type": "string", "minLength": 1, "maxLength": 20 }, "kind": { "type": "string", "enum": [ "risk", "benefit", "question", "assumption" ] }, "text": { "type": "string", "minLength": 1, "maxLength": 400 }, "why": { "type": "string", "minLength": 1, "maxLength": 400 }, "confidence": { "type": "string", "enum": [ "low", "medium", "high" ] } } } }, "would_change_my_mind": { "type": "string", "minLength": 1, "maxLength": 400 }, "sealed_at": { "type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}(:[0-9]{2})?Z$", "maxLength": 30 } } }, "rules": [ "Sealed on submission by its digest." ] } }, { "slug": "template.independence-check", "category": "noun.eval_run", "role": "template", "title": "Template: the check before anyone reads the views: digests, packets, seal times, exclusions and the reading order", "category_reason": "An independence check is a deterministic verification.", "value": { "describes": "the check before anyone reads the views: digests, packets, seal times, exclusions and the reading order", "schema": { "type": "object", "additionalProperties": false, "required": [ "panel_id", "checked_at", "views", "same_packet", "all_sealed_before_reading", "excluded", "shuffle_seed", "read_order", "result" ], "properties": { "panel_id": { "type": "string", "minLength": 1, "maxLength": 40 }, "checked_at": { "type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}(:[0-9]{2})?Z$", "maxLength": 30 }, "views": { "type": "array", "minItems": 1, "maxItems": 7, "items": { "type": "object", "additionalProperties": false, "required": [ "reviewer_id", "view_sha256", "packet_sha256", "sealed_at", "late", "names_other_reviewers" ], "properties": { "reviewer_id": { "type": "string", "minLength": 1, "maxLength": 20 }, "view_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, "packet_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, "sealed_at": { "type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}(:[0-9]{2})?Z$", "maxLength": 30 }, "late": { "type": "boolean" }, "names_other_reviewers": { "type": "boolean" } } } }, "same_packet": { "type": "boolean" }, "all_sealed_before_reading": { "type": "boolean" }, "excluded": { "type": "array", "minItems": 0, "maxItems": 7, "items": { "type": "object", "additionalProperties": false, "required": [ "reviewer_id", "reason" ], "properties": { "reviewer_id": { "type": "string", "minLength": 1, "maxLength": 20 }, "reason": { "type": "string", "minLength": 1, "maxLength": 300 } } } }, "shuffle_seed": { "type": "integer", "minimum": 0, "maximum": 2147483647 }, "read_order": { "type": "array", "minItems": 0, "maxItems": 7, "items": { "type": "string", "minLength": 1, "maxLength": 20 } }, "result": { "type": "string", "enum": [ "passed", "failed" ] } } }, "rules": [ "result is passed only when every included view used the packet, was sealed before reading and names no other reviewer, and at least three remain." ] } }, { "slug": "template.synthesis", "category": "noun.seed_hypothesis", "role": "template", "title": "Template: a synthesis: the panel and producer, the independence result, the sealed views, each disagreement with its quoted sides, kind and route, agreements, single-reviewer points, a summary and limits", "category_reason": "A synthesis is a structured hypothesis about where the views differ, for a person to weigh.", "value": { "describes": "a synthesis: the panel and producer, the independence result, the sealed views, each disagreement with its quoted sides, kind and route, agreements, single-reviewer points, a summary and limits", "schema": { "type": "object", "additionalProperties": false, "required": [ "synthesis_id", "panel_id", "question", "producer", "panel", "same_family_reviewers", "independence", "views", "disagreements", "positions", "agreements", "single_reviewer_points", "disagreement_count", "why_no_disagreement", "summary", "decision_left_to", "limits" ], "properties": { "synthesis_id": { "type": "string", "minLength": 1, "maxLength": 40 }, "panel_id": { "type": "string", "minLength": 1, "maxLength": 40 }, "question": { "type": "string", "minLength": 1, "maxLength": 400 }, "producer": { "type": "object", "additionalProperties": false, "required": [ "model_family", "on_panel" ], "properties": { "model_family": { "type": "string", "minLength": 0, "maxLength": 60 }, "on_panel": { "type": "boolean" } } }, "panel": { "type": "array", "minItems": 0, "maxItems": 7, "items": { "type": "object", "additionalProperties": false, "required": [ "reviewer_id", "perspective", "model_family" ], "properties": { "reviewer_id": { "type": "string", "minLength": 1, "maxLength": 20 }, "perspective": { "type": "string", "minLength": 0, "maxLength": 80 }, "model_family": { "type": "string", "minLength": 0, "maxLength": 60 } } } }, "same_family_reviewers": { "type": "array", "minItems": 0, "maxItems": 7, "items": { "type": "object", "additionalProperties": false, "required": [ "reviewer_id", "reason" ], "properties": { "reviewer_id": { "type": "string", "minLength": 1, "maxLength": 20 }, "reason": { "type": "string", "minLength": 0, "maxLength": 300 } } } }, "independence": { "type": "object", "additionalProperties": false, "required": [ "check_ref", "result" ], "properties": { "check_ref": { "type": "string", "minLength": 1, "maxLength": 300 }, "result": { "type": "string", "enum": [ "passed", "failed", "not_run" ] } } }, "views": { "type": "array", "minItems": 0, "maxItems": 7, "items": { "type": "object", "additionalProperties": false, "required": [ "reviewer_id", "view_sha256", "sealed_at" ], "properties": { "reviewer_id": { "type": "string", "minLength": 1, "maxLength": 20 }, "view_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, "sealed_at": { "type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}(:[0-9]{2})?Z$", "maxLength": 30 } } } }, "disagreements": { "type": "array", "minItems": 0, "maxItems": 30, "items": { "type": "object", "additionalProperties": false, "required": [ "id", "topic", "kind", "route", "would_resolve" ], "properties": { "id": { "type": "string", "minLength": 1, "maxLength": 20 }, "topic": { "type": "string", "minLength": 1, "maxLength": 300 }, "kind": { "type": "string", "enum": [ "factual", "value", "taste", "scope" ] }, "route": { "type": "string", "enum": [ "evaluation", "person", "taste_palette", "requester" ] }, "would_resolve": { "type": "string", "minLength": 0, "maxLength": 400 } } } }, "positions": { "type": "array", "minItems": 0, "maxItems": 100, "items": { "type": "object", "additionalProperties": false, "required": [ "disagreement_id", "reviewer_id", "position", "quote" ], "properties": { "disagreement_id": { "type": "string", "minLength": 1, "maxLength": 20 }, "reviewer_id": { "type": "string", "minLength": 1, "maxLength": 20 }, "position": { "type": "string", "minLength": 1, "maxLength": 300 }, "quote": { "type": "string", "minLength": 0, "maxLength": 400 } } } }, "agreements": { "type": "array", "minItems": 0, "maxItems": 20, "items": { "type": "string", "minLength": 1, "maxLength": 300 } }, "single_reviewer_points": { "type": "array", "minItems": 0, "maxItems": 30, "items": { "type": "object", "additionalProperties": false, "required": [ "reviewer_id", "point" ], "properties": { "reviewer_id": { "type": "string", "minLength": 1, "maxLength": 20 }, "point": { "type": "string", "minLength": 1, "maxLength": 400 } } } }, "disagreement_count": { "type": "integer", "minimum": 0, "maximum": 30 }, "why_no_disagreement": { "type": "string", "minLength": 0, "maxLength": 400 }, "summary": { "type": "string", "minLength": 1, "maxLength": 800 }, "decision_left_to": { "type": "string", "enum": [ "person", "panel", "synthesizer" ] }, "limits": { "type": "array", "minItems": 1, "maxItems": 10, "items": { "type": "string", "minLength": 1, "maxLength": 300 } } } }, "rules": [ "Every quote appears exactly in its reviewer's sealed view." ] } }, { "slug": "template.synthesis-audit", "category": "noun.eval_run", "role": "template", "title": "Template: an audit of a synthesis by someone who wrote none of it: quotes checked, points dropped, routes and head-count language", "category_reason": "An audit is a review against fixed rules.", "value": { "describes": "an audit of a synthesis by someone who wrote none of it: quotes checked, points dropped, routes and head-count language", "schema": { "type": "object", "additionalProperties": false, "required": [ "synthesis_id", "audited_at", "auditor_kind", "positions_checked", "quotes_not_found", "points_dropped", "misrouted", "head_count_language", "passed" ], "properties": { "synthesis_id": { "type": "string", "minLength": 1, "maxLength": 40 }, "audited_at": { "type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}(:[0-9]{2})?Z$", "maxLength": 30 }, "auditor_kind": { "type": "string", "enum": [ "different_worker", "person" ] }, "positions_checked": { "type": "integer", "minimum": 0, "maximum": 1000 }, "quotes_not_found": { "type": "array", "minItems": 0, "maxItems": 100, "items": { "type": "string", "minLength": 1, "maxLength": 60 } }, "points_dropped": { "type": "array", "minItems": 0, "maxItems": 100, "items": { "type": "string", "minLength": 1, "maxLength": 60 } }, "misrouted": { "type": "array", "minItems": 0, "maxItems": 30, "items": { "type": "object", "additionalProperties": false, "required": [ "disagreement_id", "why" ], "properties": { "disagreement_id": { "type": "string", "minLength": 1, "maxLength": 20 }, "why": { "type": "string", "minLength": 1, "maxLength": 300 } } } }, "head_count_language": { "type": "array", "minItems": 0, "maxItems": 10, "items": { "type": "string", "minLength": 1, "maxLength": 300 } }, "passed": { "type": "boolean" } } }, "rules": [ "passed is true only when every list is empty." ] } }, { "slug": "template.decision", "category": "noun.eval_run", "role": "template", "title": "Template: a person's decision on a synthesis: the decision, how each disagreement was settled or what comes next, and why", "category_reason": "A decision is a person's attested outcome.", "value": { "describes": "a person's decision on a synthesis: the decision, how each disagreement was settled or what comes next, and why", "schema": { "type": "object", "additionalProperties": false, "required": [ "synthesis_ref", "audit_ref", "audit_passed", "decided_by_role", "decided_by_kind", "decision", "settled", "left_open", "reason", "decided_on" ], "properties": { "synthesis_ref": { "type": "string", "minLength": 1, "maxLength": 300 }, "audit_ref": { "type": "string", "minLength": 1, "maxLength": 300 }, "audit_passed": { "type": "boolean" }, "decided_by_role": { "type": "string", "minLength": 1, "maxLength": 80 }, "decided_by_kind": { "type": "string", "enum": [ "person" ] }, "decision": { "type": "string", "enum": [ "proceed", "proceed_with_changes", "do_not_proceed", "decide_later" ] }, "settled": { "type": "array", "minItems": 0, "maxItems": 30, "items": { "type": "object", "additionalProperties": false, "required": [ "disagreement_id", "how" ], "properties": { "disagreement_id": { "type": "string", "minLength": 1, "maxLength": 20 }, "how": { "type": "string", "minLength": 1, "maxLength": 400 } } } }, "left_open": { "type": "array", "minItems": 0, "maxItems": 30, "items": { "type": "object", "additionalProperties": false, "required": [ "disagreement_id", "next" ], "properties": { "disagreement_id": { "type": "string", "minLength": 1, "maxLength": 20 }, "next": { "type": "string", "minLength": 1, "maxLength": 400 } } } }, "reason": { "type": "string", "minLength": 1, "maxLength": 600 }, "decided_on": { "type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" } } }, "rules": [ "Every disagreement id is in settled or left_open.", "decided_by_role names a role, never a person." ] } }, { "slug": "panel-checks", "category": "noun.benchmark_suite", "role": "record_checks", "title": "Record checks for panel syntheses, with synthetic golden syntheses", "category_reason": "A deterministic check suite with known-good and known-bad examples; the validator recomputes every verdict.", "value": { "fixture_kind": "finding_checks", "purpose": "Checks every synthesis must pass before a person relies on it. The two synthetic good syntheses, one with a disagreement and one without, pass all of them; each synthetic bad synthesis fails exactly the one check named in fails.", "deterministic_checks": [ { "id": "perspectives-stated", "rule": { "rule": "each_has", "path": "panel", "fields": [ "reviewer_id", "perspective", "model_family" ], "min": 3 }, "why": "At least three reviewers, each with a stated perspective and a recorded model family." }, { "id": "producer-family-recorded", "rule": { "rule": "present", "path": "producer.model_family" }, "why": "The producer's model family is recorded, so independence from it can be judged." }, { "id": "producer-not-on-panel", "rule": { "rule": "enum", "path": "producer.on_panel", "values": [ false ] }, "why": "The producer does not review its own work." }, { "id": "same-family-explained", "rule": { "rule": "each_has", "path": "same_family_reviewers", "fields": [ "reviewer_id", "reason" ] }, "why": "A reviewer on the producer's family is named with the reason." }, { "id": "independence-passed", "rule": { "rule": "enum", "path": "independence.result", "values": [ "passed" ] }, "why": "The synthesis rests on views the independence check passed." }, { "id": "views-sealed", "rule": { "rule": "each_has", "path": "views", "fields": [ "reviewer_id", "view_sha256", "sealed_at" ], "min": 3 }, "why": "Every view was sealed by its digest before anyone read it." }, { "id": "disagreements-routed", "rule": { "rule": "each_has", "path": "disagreements", "fields": [ "topic", "kind", "route", "would_resolve" ] }, "why": "Every disagreement has its kind, where it goes next and what would resolve it." }, { "id": "positions-quoted", "rule": { "rule": "each_has", "path": "positions", "fields": [ "disagreement_id", "reviewer_id", "position", "quote" ] }, "why": "Every side of a disagreement is quoted from its reviewer's sealed view." }, { "id": "no-disagreement-explained", "rule": { "rule": "when", "if": { "path": "disagreement_count", "equals": 0 }, "then": { "rule": "present", "path": "why_no_disagreement" } }, "why": "A panel with no disagreement says why that may be real or correlated." }, { "id": "no-head-count-verdict", "rule": { "rule": "pattern_absent", "path": "summary", "pattern": "(?i)\\b(the majority|most reviewers|outvoted|by a vote|majority view|consensus wins)\\b" }, "why": "A count of reviewers never settles a disagreement." }, { "id": "person-decides", "rule": { "rule": "enum", "path": "decision_left_to", "values": [ "person" ] }, "why": "The synthesis decides nothing; a person does." }, { "id": "limits-stated", "rule": { "rule": "min_items", "path": "limits", "min": 1 }, "why": "Every synthesis says what the panel could not show." } ], "golden_fixtures": [ { "id": "good-split", "label": "known_good", "provenance": "synthetic", "finding": { "synthesis_id": "PS-2", "panel_id": "PP-2", "question": "Adopt the new intake form?", "producer": { "model_family": "fam-a", "on_panel": false }, "panel": [ { "reviewer_id": "r1", "perspective": "served", "model_family": "fam-b" }, { "reviewer_id": "r2", "perspective": "skeptic", "model_family": "fam-c" }, { "reviewer_id": "r3", "perspective": "operator", "model_family": "fam-a" } ], "same_family_reviewers": [ { "reviewer_id": "r3", "reason": "Only two families." } ], "independence": { "check_ref": "I-2", "result": "passed" }, "views": [ { "reviewer_id": "r1", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" }, { "reviewer_id": "r2", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" }, { "reviewer_id": "r3", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" } ], "disagreements": [ { "id": "D1", "topic": "Works on old phones?", "kind": "factual", "route": "evaluation", "would_resolve": "Try five." } ], "positions": [ { "disagreement_id": "D1", "reviewer_id": "r1", "position": "No.", "quote": "It froze." }, { "disagreement_id": "D1", "reviewer_id": "r3", "position": "Yes.", "quote": "It loads." } ], "agreements": [ "Paper forms get lost." ], "single_reviewer_points": [ { "reviewer_id": "r2", "point": "Spam sign-ups." } ], "disagreement_count": 1, "why_no_disagreement": "", "summary": "One factual disagreement; a trial settles it.", "decision_left_to": "person", "limits": [ "One reviewer shares the producer's family." ] } }, { "id": "good-agreed", "label": "known_good", "provenance": "synthetic", "finding": { "synthesis_id": "PS-3", "panel_id": "PP-3", "question": "Fix the flyer typo?", "producer": { "model_family": "fam-a", "on_panel": false }, "panel": [ { "reviewer_id": "r1", "perspective": "newcomer", "model_family": "fam-b" }, { "reviewer_id": "r2", "perspective": "skeptic", "model_family": "fam-c" }, { "reviewer_id": "r3", "perspective": "steward", "model_family": "fam-b" } ], "same_family_reviewers": [], "independence": { "check_ref": "I-3", "result": "passed" }, "views": [ { "reviewer_id": "r1", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T11:00Z" }, { "reviewer_id": "r2", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T11:00Z" }, { "reviewer_id": "r3", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T11:00Z" } ], "disagreements": [], "positions": [], "agreements": [ "It changes one word." ], "single_reviewer_points": [], "disagreement_count": 0, "why_no_disagreement": "A one-word fix; two other families found nothing.", "summary": "No disagreement.", "decision_left_to": "person", "limits": [ "A small change." ] } }, { "id": "bad-no-perspective", "label": "known_bad", "fails": "perspectives-stated", "why": "A reviewer with no stated perspective.", "finding": { "synthesis_id": "PS-2", "panel_id": "PP-2", "question": "Adopt the new intake form?", "producer": { "model_family": "fam-a", "on_panel": false }, "panel": [ { "reviewer_id": "r1", "perspective": "served", "model_family": "fam-b" }, { "reviewer_id": "r2", "perspective": "", "model_family": "fam-c" }, { "reviewer_id": "r3", "perspective": "operator", "model_family": "fam-a" } ], "same_family_reviewers": [ { "reviewer_id": "r3", "reason": "Only two families." } ], "independence": { "check_ref": "I-2", "result": "passed" }, "views": [ { "reviewer_id": "r1", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" }, { "reviewer_id": "r2", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" }, { "reviewer_id": "r3", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" } ], "disagreements": [ { "id": "D1", "topic": "Works on old phones?", "kind": "factual", "route": "evaluation", "would_resolve": "Try five." } ], "positions": [ { "disagreement_id": "D1", "reviewer_id": "r1", "position": "No.", "quote": "It froze." }, { "disagreement_id": "D1", "reviewer_id": "r3", "position": "Yes.", "quote": "It loads." } ], "agreements": [ "Paper forms get lost." ], "single_reviewer_points": [ { "reviewer_id": "r2", "point": "Spam sign-ups." } ], "disagreement_count": 1, "why_no_disagreement": "", "summary": "One factual disagreement; a trial settles it.", "decision_left_to": "person", "limits": [ "One reviewer shares the producer's family." ] } }, { "id": "bad-producer-unknown", "label": "known_bad", "fails": "producer-family-recorded", "why": "The producer's family is not recorded.", "finding": { "synthesis_id": "PS-2", "panel_id": "PP-2", "question": "Adopt the new intake form?", "producer": { "model_family": "", "on_panel": false }, "panel": [ { "reviewer_id": "r1", "perspective": "served", "model_family": "fam-b" }, { "reviewer_id": "r2", "perspective": "skeptic", "model_family": "fam-c" }, { "reviewer_id": "r3", "perspective": "operator", "model_family": "fam-a" } ], "same_family_reviewers": [ { "reviewer_id": "r3", "reason": "Only two families." } ], "independence": { "check_ref": "I-2", "result": "passed" }, "views": [ { "reviewer_id": "r1", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" }, { "reviewer_id": "r2", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" }, { "reviewer_id": "r3", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" } ], "disagreements": [ { "id": "D1", "topic": "Works on old phones?", "kind": "factual", "route": "evaluation", "would_resolve": "Try five." } ], "positions": [ { "disagreement_id": "D1", "reviewer_id": "r1", "position": "No.", "quote": "It froze." }, { "disagreement_id": "D1", "reviewer_id": "r3", "position": "Yes.", "quote": "It loads." } ], "agreements": [ "Paper forms get lost." ], "single_reviewer_points": [ { "reviewer_id": "r2", "point": "Spam sign-ups." } ], "disagreement_count": 1, "why_no_disagreement": "", "summary": "One factual disagreement; a trial settles it.", "decision_left_to": "person", "limits": [ "One reviewer shares the producer's family." ] } }, { "id": "bad-self-review", "label": "known_bad", "fails": "producer-not-on-panel", "why": "The producer sat on its own panel.", "finding": { "synthesis_id": "PS-2", "panel_id": "PP-2", "question": "Adopt the new intake form?", "producer": { "model_family": "fam-a", "on_panel": true }, "panel": [ { "reviewer_id": "r1", "perspective": "served", "model_family": "fam-b" }, { "reviewer_id": "r2", "perspective": "skeptic", "model_family": "fam-c" }, { "reviewer_id": "r3", "perspective": "operator", "model_family": "fam-a" } ], "same_family_reviewers": [ { "reviewer_id": "r3", "reason": "Only two families." } ], "independence": { "check_ref": "I-2", "result": "passed" }, "views": [ { "reviewer_id": "r1", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" }, { "reviewer_id": "r2", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" }, { "reviewer_id": "r3", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" } ], "disagreements": [ { "id": "D1", "topic": "Works on old phones?", "kind": "factual", "route": "evaluation", "would_resolve": "Try five." } ], "positions": [ { "disagreement_id": "D1", "reviewer_id": "r1", "position": "No.", "quote": "It froze." }, { "disagreement_id": "D1", "reviewer_id": "r3", "position": "Yes.", "quote": "It loads." } ], "agreements": [ "Paper forms get lost." ], "single_reviewer_points": [ { "reviewer_id": "r2", "point": "Spam sign-ups." } ], "disagreement_count": 1, "why_no_disagreement": "", "summary": "One factual disagreement; a trial settles it.", "decision_left_to": "person", "limits": [ "One reviewer shares the producer's family." ] } }, { "id": "bad-same-family-silent", "label": "known_bad", "fails": "same-family-explained", "why": "A same-family reviewer with no reason.", "finding": { "synthesis_id": "PS-2", "panel_id": "PP-2", "question": "Adopt the new intake form?", "producer": { "model_family": "fam-a", "on_panel": false }, "panel": [ { "reviewer_id": "r1", "perspective": "served", "model_family": "fam-b" }, { "reviewer_id": "r2", "perspective": "skeptic", "model_family": "fam-c" }, { "reviewer_id": "r3", "perspective": "operator", "model_family": "fam-a" } ], "same_family_reviewers": [ { "reviewer_id": "r3", "reason": "" } ], "independence": { "check_ref": "I-2", "result": "passed" }, "views": [ { "reviewer_id": "r1", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" }, { "reviewer_id": "r2", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" }, { "reviewer_id": "r3", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" } ], "disagreements": [ { "id": "D1", "topic": "Works on old phones?", "kind": "factual", "route": "evaluation", "would_resolve": "Try five." } ], "positions": [ { "disagreement_id": "D1", "reviewer_id": "r1", "position": "No.", "quote": "It froze." }, { "disagreement_id": "D1", "reviewer_id": "r3", "position": "Yes.", "quote": "It loads." } ], "agreements": [ "Paper forms get lost." ], "single_reviewer_points": [ { "reviewer_id": "r2", "point": "Spam sign-ups." } ], "disagreement_count": 1, "why_no_disagreement": "", "summary": "One factual disagreement; a trial settles it.", "decision_left_to": "person", "limits": [ "One reviewer shares the producer's family." ] } }, { "id": "bad-independence-failed", "label": "known_bad", "fails": "independence-passed", "why": "Synthesized from views that failed the check.", "finding": { "synthesis_id": "PS-2", "panel_id": "PP-2", "question": "Adopt the new intake form?", "producer": { "model_family": "fam-a", "on_panel": false }, "panel": [ { "reviewer_id": "r1", "perspective": "served", "model_family": "fam-b" }, { "reviewer_id": "r2", "perspective": "skeptic", "model_family": "fam-c" }, { "reviewer_id": "r3", "perspective": "operator", "model_family": "fam-a" } ], "same_family_reviewers": [ { "reviewer_id": "r3", "reason": "Only two families." } ], "independence": { "check_ref": "I-2", "result": "failed" }, "views": [ { "reviewer_id": "r1", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" }, { "reviewer_id": "r2", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" }, { "reviewer_id": "r3", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" } ], "disagreements": [ { "id": "D1", "topic": "Works on old phones?", "kind": "factual", "route": "evaluation", "would_resolve": "Try five." } ], "positions": [ { "disagreement_id": "D1", "reviewer_id": "r1", "position": "No.", "quote": "It froze." }, { "disagreement_id": "D1", "reviewer_id": "r3", "position": "Yes.", "quote": "It loads." } ], "agreements": [ "Paper forms get lost." ], "single_reviewer_points": [ { "reviewer_id": "r2", "point": "Spam sign-ups." } ], "disagreement_count": 1, "why_no_disagreement": "", "summary": "One factual disagreement; a trial settles it.", "decision_left_to": "person", "limits": [ "One reviewer shares the producer's family." ] } }, { "id": "bad-unsealed-view", "label": "known_bad", "fails": "views-sealed", "why": "A view read before it was sealed.", "finding": { "synthesis_id": "PS-2", "panel_id": "PP-2", "question": "Adopt the new intake form?", "producer": { "model_family": "fam-a", "on_panel": false }, "panel": [ { "reviewer_id": "r1", "perspective": "served", "model_family": "fam-b" }, { "reviewer_id": "r2", "perspective": "skeptic", "model_family": "fam-c" }, { "reviewer_id": "r3", "perspective": "operator", "model_family": "fam-a" } ], "same_family_reviewers": [ { "reviewer_id": "r3", "reason": "Only two families." } ], "independence": { "check_ref": "I-2", "result": "passed" }, "views": [ { "reviewer_id": "r1", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" }, { "reviewer_id": "r2", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" }, { "reviewer_id": "r3", "view_sha256": "", "sealed_at": "2026-10-30T10:00Z" } ], "disagreements": [ { "id": "D1", "topic": "Works on old phones?", "kind": "factual", "route": "evaluation", "would_resolve": "Try five." } ], "positions": [ { "disagreement_id": "D1", "reviewer_id": "r1", "position": "No.", "quote": "It froze." }, { "disagreement_id": "D1", "reviewer_id": "r3", "position": "Yes.", "quote": "It loads." } ], "agreements": [ "Paper forms get lost." ], "single_reviewer_points": [ { "reviewer_id": "r2", "point": "Spam sign-ups." } ], "disagreement_count": 1, "why_no_disagreement": "", "summary": "One factual disagreement; a trial settles it.", "decision_left_to": "person", "limits": [ "One reviewer shares the producer's family." ] } }, { "id": "bad-unrouted", "label": "known_bad", "fails": "disagreements-routed", "why": "A disagreement with nothing that would resolve it.", "finding": { "synthesis_id": "PS-2", "panel_id": "PP-2", "question": "Adopt the new intake form?", "producer": { "model_family": "fam-a", "on_panel": false }, "panel": [ { "reviewer_id": "r1", "perspective": "served", "model_family": "fam-b" }, { "reviewer_id": "r2", "perspective": "skeptic", "model_family": "fam-c" }, { "reviewer_id": "r3", "perspective": "operator", "model_family": "fam-a" } ], "same_family_reviewers": [ { "reviewer_id": "r3", "reason": "Only two families." } ], "independence": { "check_ref": "I-2", "result": "passed" }, "views": [ { "reviewer_id": "r1", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" }, { "reviewer_id": "r2", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" }, { "reviewer_id": "r3", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" } ], "disagreements": [ { "id": "D1", "topic": "Works on old phones?", "kind": "factual", "route": "evaluation", "would_resolve": "" } ], "positions": [ { "disagreement_id": "D1", "reviewer_id": "r1", "position": "No.", "quote": "It froze." }, { "disagreement_id": "D1", "reviewer_id": "r3", "position": "Yes.", "quote": "It loads." } ], "agreements": [ "Paper forms get lost." ], "single_reviewer_points": [ { "reviewer_id": "r2", "point": "Spam sign-ups." } ], "disagreement_count": 1, "why_no_disagreement": "", "summary": "One factual disagreement; a trial settles it.", "decision_left_to": "person", "limits": [ "One reviewer shares the producer's family." ] } }, { "id": "bad-paraphrased", "label": "known_bad", "fails": "positions-quoted", "why": "One side paraphrased, not quoted.", "finding": { "synthesis_id": "PS-2", "panel_id": "PP-2", "question": "Adopt the new intake form?", "producer": { "model_family": "fam-a", "on_panel": false }, "panel": [ { "reviewer_id": "r1", "perspective": "served", "model_family": "fam-b" }, { "reviewer_id": "r2", "perspective": "skeptic", "model_family": "fam-c" }, { "reviewer_id": "r3", "perspective": "operator", "model_family": "fam-a" } ], "same_family_reviewers": [ { "reviewer_id": "r3", "reason": "Only two families." } ], "independence": { "check_ref": "I-2", "result": "passed" }, "views": [ { "reviewer_id": "r1", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" }, { "reviewer_id": "r2", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" }, { "reviewer_id": "r3", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" } ], "disagreements": [ { "id": "D1", "topic": "Works on old phones?", "kind": "factual", "route": "evaluation", "would_resolve": "Try five." } ], "positions": [ { "disagreement_id": "D1", "reviewer_id": "r1", "position": "No.", "quote": "It froze." }, { "disagreement_id": "D1", "reviewer_id": "r3", "position": "Yes.", "quote": "" } ], "agreements": [ "Paper forms get lost." ], "single_reviewer_points": [ { "reviewer_id": "r2", "point": "Spam sign-ups." } ], "disagreement_count": 1, "why_no_disagreement": "", "summary": "One factual disagreement; a trial settles it.", "decision_left_to": "person", "limits": [ "One reviewer shares the producer's family." ] } }, { "id": "bad-silent-agreement", "label": "known_bad", "fails": "no-disagreement-explained", "why": "No disagreement, and no word on why.", "finding": { "synthesis_id": "PS-3", "panel_id": "PP-3", "question": "Fix the flyer typo?", "producer": { "model_family": "fam-a", "on_panel": false }, "panel": [ { "reviewer_id": "r1", "perspective": "newcomer", "model_family": "fam-b" }, { "reviewer_id": "r2", "perspective": "skeptic", "model_family": "fam-c" }, { "reviewer_id": "r3", "perspective": "steward", "model_family": "fam-b" } ], "same_family_reviewers": [], "independence": { "check_ref": "I-3", "result": "passed" }, "views": [ { "reviewer_id": "r1", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T11:00Z" }, { "reviewer_id": "r2", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T11:00Z" }, { "reviewer_id": "r3", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T11:00Z" } ], "disagreements": [], "positions": [], "agreements": [ "It changes one word." ], "single_reviewer_points": [], "disagreement_count": 0, "why_no_disagreement": "", "summary": "No disagreement.", "decision_left_to": "person", "limits": [ "A small change." ] } }, { "id": "bad-head-count", "label": "known_bad", "fails": "no-head-count-verdict", "why": "A majority settled it.", "finding": { "synthesis_id": "PS-2", "panel_id": "PP-2", "question": "Adopt the new intake form?", "producer": { "model_family": "fam-a", "on_panel": false }, "panel": [ { "reviewer_id": "r1", "perspective": "served", "model_family": "fam-b" }, { "reviewer_id": "r2", "perspective": "skeptic", "model_family": "fam-c" }, { "reviewer_id": "r3", "perspective": "operator", "model_family": "fam-a" } ], "same_family_reviewers": [ { "reviewer_id": "r3", "reason": "Only two families." } ], "independence": { "check_ref": "I-2", "result": "passed" }, "views": [ { "reviewer_id": "r1", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" }, { "reviewer_id": "r2", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" }, { "reviewer_id": "r3", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" } ], "disagreements": [ { "id": "D1", "topic": "Works on old phones?", "kind": "factual", "route": "evaluation", "would_resolve": "Try five." } ], "positions": [ { "disagreement_id": "D1", "reviewer_id": "r1", "position": "No.", "quote": "It froze." }, { "disagreement_id": "D1", "reviewer_id": "r3", "position": "Yes.", "quote": "It loads." } ], "agreements": [ "Paper forms get lost." ], "single_reviewer_points": [ { "reviewer_id": "r2", "point": "Spam sign-ups." } ], "disagreement_count": 1, "why_no_disagreement": "", "summary": "The majority says it works, so it does.", "decision_left_to": "person", "limits": [ "One reviewer shares the producer's family." ] } }, { "id": "bad-synthesizer-decides", "label": "known_bad", "fails": "person-decides", "why": "The synthesis made the decision.", "finding": { "synthesis_id": "PS-2", "panel_id": "PP-2", "question": "Adopt the new intake form?", "producer": { "model_family": "fam-a", "on_panel": false }, "panel": [ { "reviewer_id": "r1", "perspective": "served", "model_family": "fam-b" }, { "reviewer_id": "r2", "perspective": "skeptic", "model_family": "fam-c" }, { "reviewer_id": "r3", "perspective": "operator", "model_family": "fam-a" } ], "same_family_reviewers": [ { "reviewer_id": "r3", "reason": "Only two families." } ], "independence": { "check_ref": "I-2", "result": "passed" }, "views": [ { "reviewer_id": "r1", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" }, { "reviewer_id": "r2", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" }, { "reviewer_id": "r3", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" } ], "disagreements": [ { "id": "D1", "topic": "Works on old phones?", "kind": "factual", "route": "evaluation", "would_resolve": "Try five." } ], "positions": [ { "disagreement_id": "D1", "reviewer_id": "r1", "position": "No.", "quote": "It froze." }, { "disagreement_id": "D1", "reviewer_id": "r3", "position": "Yes.", "quote": "It loads." } ], "agreements": [ "Paper forms get lost." ], "single_reviewer_points": [ { "reviewer_id": "r2", "point": "Spam sign-ups." } ], "disagreement_count": 1, "why_no_disagreement": "", "summary": "One factual disagreement; a trial settles it.", "decision_left_to": "synthesizer", "limits": [ "One reviewer shares the producer's family." ] } }, { "id": "bad-no-limits", "label": "known_bad", "fails": "limits-stated", "why": "No limits.", "finding": { "synthesis_id": "PS-2", "panel_id": "PP-2", "question": "Adopt the new intake form?", "producer": { "model_family": "fam-a", "on_panel": false }, "panel": [ { "reviewer_id": "r1", "perspective": "served", "model_family": "fam-b" }, { "reviewer_id": "r2", "perspective": "skeptic", "model_family": "fam-c" }, { "reviewer_id": "r3", "perspective": "operator", "model_family": "fam-a" } ], "same_family_reviewers": [ { "reviewer_id": "r3", "reason": "Only two families." } ], "independence": { "check_ref": "I-2", "result": "passed" }, "views": [ { "reviewer_id": "r1", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" }, { "reviewer_id": "r2", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" }, { "reviewer_id": "r3", "view_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "sealed_at": "2026-10-30T10:00Z" } ], "disagreements": [ { "id": "D1", "topic": "Works on old phones?", "kind": "factual", "route": "evaluation", "would_resolve": "Try five." } ], "positions": [ { "disagreement_id": "D1", "reviewer_id": "r1", "position": "No.", "quote": "It froze." }, { "disagreement_id": "D1", "reviewer_id": "r3", "position": "Yes.", "quote": "It loads." } ], "agreements": [ "Paper forms get lost." ], "single_reviewer_points": [ { "reviewer_id": "r2", "point": "Spam sign-ups." } ], "disagreement_count": 1, "why_no_disagreement": "", "summary": "One factual disagreement; a trial settles it.", "decision_left_to": "person", "limits": [] } } ], "finding_schema": { "type": "object", "additionalProperties": false, "required": [ "synthesis_id", "panel_id", "question", "producer", "panel", "same_family_reviewers", "independence", "views", "disagreements", "positions", "agreements", "single_reviewer_points", "disagreement_count", "why_no_disagreement", "summary", "decision_left_to", "limits" ], "properties": { "synthesis_id": { "type": "string", "minLength": 1, "maxLength": 40 }, "panel_id": { "type": "string", "minLength": 1, "maxLength": 40 }, "question": { "type": "string", "minLength": 1, "maxLength": 400 }, "producer": { "type": "object", "additionalProperties": false, "required": [ "model_family", "on_panel" ], "properties": { "model_family": { "type": "string", "minLength": 0, "maxLength": 60 }, "on_panel": { "type": "boolean" } } }, "panel": { "type": "array", "minItems": 0, "maxItems": 7, "items": { "type": "object", "additionalProperties": false, "required": [ "reviewer_id", "perspective", "model_family" ], "properties": { "reviewer_id": { "type": "string", "minLength": 1, "maxLength": 20 }, "perspective": { "type": "string", "minLength": 0, "maxLength": 80 }, "model_family": { "type": "string", "minLength": 0, "maxLength": 60 } } } }, "same_family_reviewers": { "type": "array", "minItems": 0, "maxItems": 7, "items": { "type": "object", "additionalProperties": false, "required": [ "reviewer_id", "reason" ], "properties": { "reviewer_id": { "type": "string", "minLength": 1, "maxLength": 20 }, "reason": { "type": "string", "minLength": 0, "maxLength": 300 } } } }, "independence": { "type": "object", "additionalProperties": false, "required": [ "check_ref", "result" ], "properties": { "check_ref": { "type": "string", "minLength": 1, "maxLength": 300 }, "result": { "type": "string", "enum": [ "passed", "failed", "not_run" ] } } }, "views": { "type": "array", "minItems": 0, "maxItems": 7, "items": { "type": "object", "additionalProperties": false, "required": [ "reviewer_id", "view_sha256", "sealed_at" ], "properties": { "reviewer_id": { "type": "string", "minLength": 1, "maxLength": 20 }, "view_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, "sealed_at": { "type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}(:[0-9]{2})?Z$", "maxLength": 30 } } } }, "disagreements": { "type": "array", "minItems": 0, "maxItems": 30, "items": { "type": "object", "additionalProperties": false, "required": [ "id", "topic", "kind", "route", "would_resolve" ], "properties": { "id": { "type": "string", "minLength": 1, "maxLength": 20 }, "topic": { "type": "string", "minLength": 1, "maxLength": 300 }, "kind": { "type": "string", "enum": [ "factual", "value", "taste", "scope" ] }, "route": { "type": "string", "enum": [ "evaluation", "person", "taste_palette", "requester" ] }, "would_resolve": { "type": "string", "minLength": 0, "maxLength": 400 } } } }, "positions": { "type": "array", "minItems": 0, "maxItems": 100, "items": { "type": "object", "additionalProperties": false, "required": [ "disagreement_id", "reviewer_id", "position", "quote" ], "properties": { "disagreement_id": { "type": "string", "minLength": 1, "maxLength": 20 }, "reviewer_id": { "type": "string", "minLength": 1, "maxLength": 20 }, "position": { "type": "string", "minLength": 1, "maxLength": 300 }, "quote": { "type": "string", "minLength": 0, "maxLength": 400 } } } }, "agreements": { "type": "array", "minItems": 0, "maxItems": 20, "items": { "type": "string", "minLength": 1, "maxLength": 300 } }, "single_reviewer_points": { "type": "array", "minItems": 0, "maxItems": 30, "items": { "type": "object", "additionalProperties": false, "required": [ "reviewer_id", "point" ], "properties": { "reviewer_id": { "type": "string", "minLength": 1, "maxLength": 20 }, "point": { "type": "string", "minLength": 1, "maxLength": 400 } } } }, "disagreement_count": { "type": "integer", "minimum": 0, "maximum": 30 }, "why_no_disagreement": { "type": "string", "minLength": 0, "maxLength": 400 }, "summary": { "type": "string", "minLength": 1, "maxLength": 800 }, "decision_left_to": { "type": "string", "enum": [ "person", "panel", "synthesizer" ] }, "limits": { "type": "array", "minItems": 1, "maxItems": 10, "items": { "type": "string", "minLength": 1, "maxLength": 300 } } } }, "expected": { "good-split": { "verdict": "pass", "failed_checks": [] }, "good-agreed": { "verdict": "pass", "failed_checks": [] }, "bad-no-perspective": { "verdict": "fail", "failed_checks": [ "perspectives-stated" ] }, "bad-producer-unknown": { "verdict": "fail", "failed_checks": [ "producer-family-recorded" ] }, "bad-self-review": { "verdict": "fail", "failed_checks": [ "producer-not-on-panel" ] }, "bad-same-family-silent": { "verdict": "fail", "failed_checks": [ "same-family-explained" ] }, "bad-independence-failed": { "verdict": "fail", "failed_checks": [ "independence-passed" ] }, "bad-unsealed-view": { "verdict": "fail", "failed_checks": [ "views-sealed" ] }, "bad-unrouted": { "verdict": "fail", "failed_checks": [ "disagreements-routed" ] }, "bad-paraphrased": { "verdict": "fail", "failed_checks": [ "positions-quoted" ] }, "bad-silent-agreement": { "verdict": "fail", "failed_checks": [ "no-disagreement-explained" ] }, "bad-head-count": { "verdict": "fail", "failed_checks": [ "no-head-count-verdict" ] }, "bad-synthesizer-decides": { "verdict": "fail", "failed_checks": [ "person-decides" ] }, "bad-no-limits": { "verdict": "fail", "failed_checks": [ "limits-stated" ] } } } }, { "slug": "checklist", "category": "noun.skill_pack", "role": "checklist", "title": "Checklist a person runs before deciding on a panel's synthesis", "category_reason": "A short checklist a person runs is part of the panel's guardrails.", "value": { "items": [ "Read the disagreements before the agreements.", "For any position you will act on, read its quote in the sealed view.", "Weigh a point by its reasons, not by how many reviewers raised it.", "Note which reviewers share the producer's model family; their agreement may be correlated.", "Settle each disagreement, or leave it open with its next step; a factual one can wait for its test.", "Deciding later is a valid decision." ] } }, { "slug": "test-cases", "category": "noun.benchmark_suite", "role": "test_cases", "title": "Synthetic test cases for persona panels", "synthetic": true, "value": { "cases": [ { "id": "PP-PRODUCER-ASKS", "scenario": "The worker that drafted the plan asks to sit on its panel", "sample_input": "SYNTHETIC: the reviewer key matches the draft's producer key.", "expected_outcome": "blocked", "expected_indicators": [ "producer_excluded" ], "expected_human_escalation": false }, { "id": "PP-PEEKED", "scenario": "A view quotes another reviewer's take", "sample_input": "SYNTHETIC: r2's view says that r1 was right about the cost.", "expected_outcome": "independence_checked", "expected_indicators": [ "view_excluded", "give_view_proposed" ], "expected_human_escalation": false }, { "id": "PP-ONE-FAMILY", "scenario": "Only the producer's model family is available", "sample_input": "SYNTHETIC: the allowance covers one family, the producer's.", "expected_outcome": "panel_defined", "expected_indicators": [ "same_family_reasons", "correlated_label" ], "expected_human_escalation": true }, { "id": "PP-ALL-AGREE", "scenario": "Every reviewer supports the plan", "sample_input": "SYNTHETIC: three supporting views with no conflicting point.", "expected_outcome": "disagreements_synthesized", "expected_indicators": [ "why_no_disagreement" ], "expected_human_escalation": false }, { "id": "PP-FACT-SPLIT", "scenario": "Reviewers disagree about something measurable", "sample_input": "SYNTHETIC: one says the form takes a minute, another says five.", "expected_outcome": "disagreements_synthesized", "expected_indicators": [ "route_evaluation", "design_evaluation_proposed" ], "expected_human_escalation": false }, { "id": "PP-TASTE-SPLIT", "scenario": "Reviewers disagree about tone", "sample_input": "SYNTHETIC: one finds the letter warm, another too casual.", "expected_outcome": "disagreements_synthesized", "expected_indicators": [ "route_taste_palette", "palette_brief" ], "expected_human_escalation": false }, { "id": "PP-HEAD-COUNT", "scenario": "A draft synthesis settles a disagreement by counting", "sample_input": "SYNTHETIC: the summary says most reviewers favor launching.", "expected_outcome": "synthesis_audited", "expected_indicators": [ "head_count_language", "audit_failed" ], "expected_human_escalation": false } ] } }, { "slug": "sources", "category": "noun.public_source", "role": "sources", "title": "Dated public sources for persona panels and second opinions", "value": { "sources": [ { "title": "Zheng and others, Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena, 2023", "url": "https://arxiv.org/abs/2306.05685", "used_for": "Position, verbosity and self-enhancement biases in model judges", "checked_date": "2026-10-30" }, { "title": "Panickssery, Bowman and Feng, LLM Evaluators Recognize and Favor Their Own Generations, 2024", "url": "https://arxiv.org/abs/2404.13076", "used_for": "Why reviewers come from a model family other than the producer's", "checked_date": "2026-10-30" }, { "title": "Wang and others, Large Language Models are not Fair Evaluators, 2023", "url": "https://arxiv.org/abs/2305.17926", "used_for": "Why views are read in a fixed shuffled order", "checked_date": "2026-10-30" }, { "title": "Klein, Performing a Project Premortem, Harvard Business Review, 2007", "url": "https://hbr.org/2007/09/performing-a-project-premortem", "used_for": "Imagining failure first, the skeptic's lens", "checked_date": "2026-10-30" } ], "recheck": "Recheck each source before relying on it; terms, limits and versions change." } }, { "slug": "example.panel", "category": "noun.seed_hypothesis", "role": "example", "title": "Worked example: a three-reviewer panel on a tablet check-in plan", "synthetic": true, "category_reason": "A synthetic instance in the template's shape.", "value": { "conforms_to": { "$pin": "template.panel" }, "instance": { "panel_id": "PP-1", "question": "Should the program replace paper sign-in sheets with tablet check-in at both tutoring sites?", "artifact": { "ref": "plan TC-1, version 2", "sha256": "0872fd06f380f480dcb9eb65f3ba53662cc9ae1e20a9c420a7d5ee3d1435e090" }, "producer": { "kind": "model_run", "model_family": "family-a", "on_panel": false }, "reviewers": [ { "reviewer_id": "r1", "perspective": "the person it serves", "card_ref": "perspective.served-person", "model_family": "family-b", "same_family_reason": "" }, { "reviewer_id": "r2", "perspective": "the skeptic", "card_ref": "perspective.skeptic", "model_family": "family-c", "same_family_reason": "" }, { "reviewer_id": "r3", "perspective": "the operator", "card_ref": "perspective.operator", "model_family": "family-a", "same_family_reason": "Only two families other than the producer's fit the worker's allowance this week." } ], "packet_sha256": "e8d59bc9d6fafe27f9909c3e681b678ae25b334fa85598bfc60f6f8b013ed05c", "deadline": "2026-10-30T12:00Z", "shuffle_seed": 20261030, "decision_owner_role": "program director" }, "shows": "Three perspectives on three families; the operator shares the producer's family, with the reason.", "source": "Synthetic. Written for this pack; every name, number and identifier is made up." } }, { "slug": "example.view-r1", "category": "noun.eval_run", "role": "example", "title": "Worked example: the view of the person it serves", "synthetic": true, "category_reason": "A synthetic instance in the template's shape.", "value": { "conforms_to": { "$pin": "template.view" }, "instance": { "panel_id": "PP-1", "reviewer_id": "r1", "perspective": "the person it serves", "model_family": "family-b", "packet_sha256": "e8d59bc9d6fafe27f9909c3e681b678ae25b334fa85598bfc60f6f8b013ed05c", "saw_other_views": false, "stance": "support_with_changes", "summary": "A tablet is quicker than a clipboard, but learners who cannot read the screen yet will be embarrassed at the door. Keep paper beside it and let a tutor check people in.", "points": [ { "id": "r1-p1", "kind": "risk", "text": "Learners who cannot read the screen yet are embarrassed in front of others.", "why": "Signing in is the first thing a learner does in public at each visit.", "confidence": "high" }, { "id": "r1-p2", "kind": "benefit", "text": "No more waiting in line to sign a sheet.", "why": "The line forms at the start of every session.", "confidence": "medium" } ], "would_change_my_mind": "Watching five learners check in on it without help.", "sealed_at": "2026-10-30T10:12Z" }, "shows": "Supports the plan with changes: keep paper beside the tablet, as some learners cannot read the screen yet.", "source": "Synthetic. Written for this pack; every name, number and identifier is made up." } }, { "slug": "example.view-r2", "category": "noun.eval_run", "role": "example", "title": "Worked example: the skeptic's view", "synthetic": true, "category_reason": "A synthetic instance in the template's shape.", "value": { "conforms_to": { "$pin": "template.view" }, "instance": { "panel_id": "PP-1", "reviewer_id": "r2", "perspective": "the skeptic", "model_family": "family-c", "packet_sha256": "e8d59bc9d6fafe27f9909c3e681b678ae25b334fa85598bfc60f6f8b013ed05c", "saw_other_views": false, "stance": "oppose", "summary": "The plan rests on wifi at both sites and on tablets nobody has budgeted to replace. The east site's wifi drops most evenings, so check-ins would be lost.", "points": [ { "id": "r2-p1", "kind": "risk", "text": "The east site's wifi drops most evenings, so check-ins would be lost.", "why": "Evening sessions are when most learners come.", "confidence": "high" }, { "id": "r2-p2", "kind": "assumption", "text": "The plan assumes funders accept tablet counts in place of signed sheets.", "why": "Attendance reports go to two funders.", "confidence": "medium" } ], "would_change_my_mind": "A week of offline check-ins that sync without loss, and a funder's written yes.", "sealed_at": "2026-10-30T10:31Z" }, "shows": "Opposes the plan: the east site's evening wifi outages would lose check-ins.", "source": "Synthetic. Written for this pack; every name, number and identifier is made up." } }, { "slug": "example.view-r3", "category": "noun.eval_run", "role": "example", "title": "Worked example: the operator's view", "synthetic": true, "category_reason": "A synthetic instance in the template's shape.", "value": { "conforms_to": { "$pin": "template.view" }, "instance": { "panel_id": "PP-1", "reviewer_id": "r3", "perspective": "the operator", "model_family": "family-a", "packet_sha256": "e8d59bc9d6fafe27f9909c3e681b678ae25b334fa85598bfc60f6f8b013ed05c", "saw_other_views": false, "stance": "support", "summary": "Coordinators spend about an hour a week typing up sign-in sheets, and the check-in app keeps check-ins on the tablet and syncs them when wifi returns.", "points": [ { "id": "r3-p1", "kind": "benefit", "text": "No weekly typing of sign-in sheets.", "why": "About an hour a week per site goes to it now.", "confidence": "high" }, { "id": "r3-p2", "kind": "benefit", "text": "The app keeps check-ins on the tablet and syncs them when wifi returns.", "why": "Its settings page lists offline mode.", "confidence": "medium" }, { "id": "r3-p3", "kind": "question", "text": "Who resets the tablet when a learner gets stuck?", "why": "Coordinators are often with a learner when the door is busy.", "confidence": "medium" } ], "would_change_my_mind": "If checking in takes longer than signing the sheet.", "sealed_at": "2026-10-30T10:44Z" }, "shows": "Supports the plan: it saves typing, and the app keeps check-ins offline.", "source": "Synthetic. Written for this pack; every name, number and identifier is made up." } }, { "slug": "example.independence-check", "category": "noun.eval_run", "role": "example", "title": "Worked example: three views checked and ordered before anyone read them", "synthetic": true, "category_reason": "A synthetic instance in the template's shape.", "value": { "conforms_to": { "$pin": "template.independence-check" }, "instance": { "panel_id": "PP-1", "checked_at": "2026-10-30T12:05Z", "views": [ { "reviewer_id": "r1", "view_sha256": "2be24e0fb217a3e098beb8f877e99067506d3c3a35348953676f64f8bfb8e7f5", "packet_sha256": "e8d59bc9d6fafe27f9909c3e681b678ae25b334fa85598bfc60f6f8b013ed05c", "sealed_at": "2026-10-30T10:12Z", "late": false, "names_other_reviewers": false }, { "reviewer_id": "r2", "view_sha256": "8edfba73ec5e75c099b0e9d8e7bb4a7e56f01d192ac9b3a954f032bf7b4fac8a", "packet_sha256": "e8d59bc9d6fafe27f9909c3e681b678ae25b334fa85598bfc60f6f8b013ed05c", "sealed_at": "2026-10-30T10:31Z", "late": false, "names_other_reviewers": false }, { "reviewer_id": "r3", "view_sha256": "7ed120073e5e8ded288e99c4041f0fba906d79c97c452689ca2111640a8f524e", "packet_sha256": "e8d59bc9d6fafe27f9909c3e681b678ae25b334fa85598bfc60f6f8b013ed05c", "sealed_at": "2026-10-30T10:44Z", "late": false, "names_other_reviewers": false } ], "same_packet": true, "all_sealed_before_reading": true, "excluded": [], "shuffle_seed": 20261030, "read_order": [ "r1", "r2", "r3" ], "result": "passed" }, "shows": "Each digest is the SHA-256 of its view; all used one packet and were sealed before the deadline.", "source": "Synthetic. Written for this pack; every name, number and identifier is made up." } }, { "slug": "example.synthesis", "category": "noun.seed_hypothesis", "role": "example", "title": "Worked example: two disagreements, one factual and one of value", "synthetic": true, "category_reason": "A synthetic instance in the template's shape.", "value": { "conforms_to": { "$pin": "template.synthesis" }, "instance": { "synthesis_id": "PS-1", "panel_id": "PP-1", "question": "Should the program replace paper sign-in sheets with tablet check-in at both tutoring sites?", "producer": { "model_family": "family-a", "on_panel": false }, "panel": [ { "reviewer_id": "r1", "perspective": "the person it serves", "model_family": "family-b" }, { "reviewer_id": "r2", "perspective": "the skeptic", "model_family": "family-c" }, { "reviewer_id": "r3", "perspective": "the operator", "model_family": "family-a" } ], "same_family_reviewers": [ { "reviewer_id": "r3", "reason": "Only two families other than the producer's fit the worker's allowance this week." } ], "independence": { "check_ref": "independence check of PP-1, 2026-10-30", "result": "passed" }, "views": [ { "reviewer_id": "r1", "view_sha256": "2be24e0fb217a3e098beb8f877e99067506d3c3a35348953676f64f8bfb8e7f5", "sealed_at": "2026-10-30T10:12Z" }, { "reviewer_id": "r2", "view_sha256": "8edfba73ec5e75c099b0e9d8e7bb4a7e56f01d192ac9b3a954f032bf7b4fac8a", "sealed_at": "2026-10-30T10:31Z" }, { "reviewer_id": "r3", "view_sha256": "7ed120073e5e8ded288e99c4041f0fba906d79c97c452689ca2111640a8f524e", "sealed_at": "2026-10-30T10:44Z" } ], "disagreements": [ { "id": "D1", "topic": "Whether check-ins at the east site survive its evening wifi outages", "kind": "factual", "route": "evaluation", "would_resolve": "A week of check-ins at the east site with wifi off for one evening, counting any lost." }, { "id": "D2", "topic": "Whether quicker check-in is worth asking learners to use a screen in public", "kind": "value", "route": "person", "would_resolve": "The program director weighs coordinators' time against learners' comfort at the door." } ], "positions": [ { "disagreement_id": "D1", "reviewer_id": "r2", "position": "Check-ins would be lost on outage evenings.", "quote": "The east site's wifi drops most evenings, so check-ins would be lost." }, { "disagreement_id": "D1", "reviewer_id": "r3", "position": "The app holds check-ins until wifi returns.", "quote": "The app keeps check-ins on the tablet and syncs them when wifi returns." }, { "disagreement_id": "D2", "reviewer_id": "r1", "position": "The screen embarrasses learners who cannot read it yet.", "quote": "Learners who cannot read the screen yet are embarrassed in front of others." }, { "disagreement_id": "D2", "reviewer_id": "r3", "position": "Coordinators' typing time is worth saving.", "quote": "No weekly typing of sign-in sheets." } ], "agreements": [ "Signing in on paper costs time, at the door and afterwards." ], "single_reviewer_points": [ { "reviewer_id": "r1", "point": "Keep paper beside the tablet and let a tutor check people in." }, { "reviewer_id": "r2", "point": "The plan assumes funders accept tablet counts in place of signed sheets." }, { "reviewer_id": "r3", "point": "Who resets the tablet when a learner gets stuck?" } ], "disagreement_count": 2, "why_no_disagreement": "", "summary": "Two disagreements. Whether check-ins survive the east site's evening outages is a factual question a week-long test can settle. Whether quicker check-in is worth asking learners to use a screen in public is a value question for the program director. One reviewer shares the producer's model family, so part of the panel is correlated.", "decision_left_to": "person", "limits": [ "Three synthetic reviewers; no learner, coordinator or funder was asked.", "r3 shares the producer's model family, so its agreement with the plan may be correlated with it." ] }, "shows": "Each side quoted from its view; the factual question goes to a test and the value one to the person.", "source": "Synthetic. Written for this pack; every name, number and identifier is made up." } }, { "slug": "example.synthesis-audit", "category": "noun.eval_run", "role": "example", "title": "Worked example: an audit that finds every quote and nothing dropped", "synthetic": true, "category_reason": "A synthetic instance in the template's shape.", "value": { "conforms_to": { "$pin": "template.synthesis-audit" }, "instance": { "synthesis_id": "PS-1", "audited_at": "2026-10-30T14:10Z", "auditor_kind": "different_worker", "positions_checked": 4, "quotes_not_found": [], "points_dropped": [], "misrouted": [], "head_count_language": [], "passed": true }, "shows": "Four positions checked against their sealed views; no point dropped, misrouted or settled by a count.", "source": "Synthetic. Written for this pack; every name, number and identifier is made up." } }, { "slug": "example.decision", "category": "noun.eval_run", "role": "example", "title": "Worked example: the program director proceeds with changes", "synthetic": true, "category_reason": "A synthetic instance in the template's shape.", "value": { "conforms_to": { "$pin": "template.decision" }, "instance": { "synthesis_ref": "synthesis PS-1", "audit_ref": "synthesis audit of PS-1, 2026-10-30", "audit_passed": true, "decided_by_role": "program director", "decided_by_kind": "person", "decision": "proceed_with_changes", "settled": [ { "disagreement_id": "D2", "how": "Keep paper sign-in beside the tablet; tutors may check learners in." } ], "left_open": [ { "disagreement_id": "D1", "next": "Run the offline test at the east site for a week before buying a second tablet." } ], "reason": "The time saved is real, but no learner should have to use the screen, and the east site's outages need testing first.", "decided_on": "2026-10-30" }, "shows": "One disagreement settled with paper kept beside the tablet; the factual one left open for its test.", "source": "Synthetic. Written for this pack; every name, number and identifier is made up." } } ], "verbs": [ { "slug": "define-panel", "kind": "verb.problemize", "title": "Define a panel of independent reviewers", "claimant": "agent", "summary": "From the plan or draft under review and the decision it informs, record who produced it and on which model family, pick three to five perspective cards, give each reviewer a family other than the producer's where one is available, and fix the shared packet, deadline and reading seed.", "outcome": "A panel, with one give-view verb proposed per reviewer.", "ready_outcome": "panel_defined", "pinned": [ "rules.panel-composition", "guardrail.different-family-where-possible", "guardrail.independent-takes", "perspective.served-person", "perspective.skeptic", "perspective.operator", "perspective.steward", "perspective.newcomer", "perspective.safety-and-privacy" ], "slots": [ { "name": "artifact", "category": "noun.seed_hypothesis", "description": "Pin of the plan, draft or proposal under review." }, { "name": "request", "category": "noun.seed_hypothesis", "description": "Pin of the requester's question: the decision the panel informs." } ], "params": { "reviewers": { "schema": { "type": "integer", "minimum": 3, "maximum": 5 }, "description": "How many reviewers.", "default": 3 } }, "produces": [ { "name": "panel", "template": "template.panel" } ], "steps": [ "Verify the pins, and record who or what produced the artifact and its model family.", "Pick perspective cards that differ, including one that looks for how the plan fails and one for the people it serves; write a new card in the same shape only when none fits.", "Give each reviewer a family other than the producer's where the worker's allowance offers one, and name any same-family reviewer with the reason.", "Fix the packet digest, a deadline and a reading seed, and propose one give-view verb per reviewer, each a separate attempt." ], "acceptance": [ { "id": "matches-template", "check": "The panel validates against template.panel's schema.", "method": "deterministic" }, { "id": "producer-off-panel", "check": "producer.on_panel is false and no reviewer is the producer.", "method": "deterministic" }, { "id": "families-recorded", "check": "Every reviewer has a model family, and each one matching the producer's has a same_family_reason.", "method": "deterministic" }, { "id": "perspectives-differ", "check": "No two reviewers share a perspective.", "method": "deterministic" } ], "ready_requires": [ "proposed_verbs" ], "bounds": { "max_elapsed_seconds": 900 } }, { "slug": "give-view", "kind": "verb.verify_gate", "title": "Give one independent view", "claimant": "agent_or_human", "review": "distinct_reviewer", "summary": "As one reviewer, read only the shared packet and your perspective card and give your own take: a stance, the points behind it with how sure you are of each, and what would change your mind.", "outcome": "One reviewer's view, sealed on submission.", "ready_outcome": "view_given", "pinned": [ "guardrail.independent-takes", "rules.independence" ], "slots": [ { "name": "panel", "category": "noun.seed_hypothesis", "description": "Pin of the panel." }, { "name": "artifact", "category": "noun.seed_hypothesis", "description": "Pin of the artifact under review." }, { "name": "card", "category": "noun.prompt_shim", "description": "Pin of this reviewer's perspective card." } ], "params": { "reviewer_id": { "schema": { "type": "string", "pattern": "^[a-z][a-z0-9-]{0,19}$" }, "description": "This reviewer's id in the panel.", "required": true } }, "produces": [ { "name": "view", "template": "template.view" } ], "steps": [ "Verify the pins, that the packet digest matches the panel's, and that you did not produce the artifact.", "Read nothing but the packet and your card: no other view, no draft synthesis and no record of the producer's own assessment.", "Give a stance and up to ten points from your perspective, each with why and how sure you are, and say what would change your mind.", "Submit once, with saw_other_views false only when that is true; the view is sealed by its digest." ], "acceptance": [ { "id": "matches-template", "check": "The view validates against template.view's schema.", "method": "deterministic" }, { "id": "same-packet", "check": "packet_sha256 equals the panel's packet_sha256.", "method": "deterministic" }, { "id": "not-the-producer", "check": "The submitting key is not the producer's or another reviewer's.", "method": "deterministic" }, { "id": "in-perspective", "check": "The points follow from the card's perspective, not a generic review.", "method": "judgment" } ], "bounds": { "max_elapsed_seconds": 1800 }, "follows": [ "persona-panel-second-opinion.define-panel" ] }, { "slug": "check-independence", "kind": "verb.verify_gate", "title": "Check the views' independence before anyone reads them", "claimant": "agent", "model_use": "none", "summary": "Before any view is read for its content, confirm that each used the panel's packet, was sealed before the deadline and names no other reviewer, record each view's digest, exclude any whose independence is lost, and fix the reading order from the seed.", "outcome": "An independence check that passes or says why not.", "ready_outcome": "independence_checked", "pinned": [ "rules.independence", "guardrail.independent-takes", "taste-palettes:method" ], "slots": [ { "name": "panel", "category": "noun.seed_hypothesis", "description": "Pin of the panel." }, { "name": "views", "category": "noun.eval_run", "description": "Pins of the sealed views.", "many": true, "max": 7 } ], "produces": [ { "name": "independence_check", "template": "template.independence-check" } ], "steps": [ "Verify the pins, and compute each view's SHA-256 over its canonical JSON.", "Compare each view's packet digest with the panel's and each seal time with the deadline, marking late views.", "Scan each view for other reviewers' ids and perspectives without weighing any view; exclude one that names or quotes another and propose a fresh give-view for that reviewer.", "Order the reviewer ids by the SHA-256 of the seed, a colon and the id, and pass only when at least three independent views remain." ], "acceptance": [ { "id": "matches-template", "check": "The check validates against template.independence-check's schema.", "method": "deterministic" }, { "id": "digests-recompute", "check": "Every view_sha256 is the SHA-256 of that view's canonical JSON.", "method": "deterministic" }, { "id": "order-follows-seed", "check": "read_order is the included reviewer ids sorted by the SHA-256 of seed:reviewer_id.", "method": "deterministic" }, { "id": "result-follows", "check": "result is passed only when same_packet and all_sealed_before_reading are true, no included view names another reviewer and at least three remain.", "method": "deterministic" } ], "bounds": { "max_elapsed_seconds": 600 }, "follows": [ "persona-panel-second-opinion.give-view" ] }, { "slug": "synthesize-disagreements", "kind": "verb.solve_draft", "title": "Synthesize where the reviewers disagree", "claimant": "agent", "review": "distinct_reviewer", "summary": "Read the sealed views in the fixed order and write a synthesis that leads with where reviewers disagree: each side quoted exactly with its reviewer and perspective, the kind of disagreement and where it goes next, then a short list of agreements and every point only one reviewer raised.", "outcome": "A synthesis that decides nothing, with palette briefs for disagreements of taste.", "ready_outcome": "disagreements_synthesized", "pinned": [ "guardrail.disagreement-first", "rules.synthesis", "panel-checks", "taste-palettes:method", "evaluation-design:method" ], "slots": [ { "name": "panel", "category": "noun.seed_hypothesis", "description": "Pin of the panel." }, { "name": "independence_check", "category": "noun.eval_run", "description": "Pin of the passed independence check." }, { "name": "views", "category": "noun.eval_run", "description": "Pins of the sealed views.", "many": true, "max": 7 } ], "produces": [ { "name": "synthesis", "template": "template.synthesis" }, { "name": "palette_brief", "template": "taste-palettes:template.brief", "max": 2 } ], "steps": [ "Verify the pins; end blocked when the independence check did not pass, and confirm you neither produced the artifact nor wrote a view.", "For each disagreement, quote every side exactly from its sealed view, name its kind and route, and say what would resolve it.", "Propose the evaluation design pack's design-evaluation verb for a factual disagreement and write a palette brief for one of taste; a value disagreement goes to the person and one of scope to the requester.", "List agreements briefly and every single-reviewer point; never turn a count of reviewers into a verdict, and run the panel checks before submitting." ], "acceptance": [ { "id": "matches-template", "check": "The synthesis validates against template.synthesis's schema.", "method": "deterministic" }, { "id": "panel-checks-pass", "check": "The synthesis passes every check in panel-checks.", "method": "deterministic" }, { "id": "quotes-exact", "check": "Every quote appears exactly in its reviewer's sealed view.", "method": "deterministic" }, { "id": "nothing-dropped", "check": "Every point in every view appears in a position, an agreement or a single-reviewer point.", "method": "judgment" } ], "bounds": { "max_elapsed_seconds": 1800 }, "follows": [ "persona-panel-second-opinion.check-independence" ] }, { "slug": "audit-synthesis", "kind": "verb.verify_gate", "title": "Audit the synthesis (someone who wrote none of it)", "claimant": "agent_or_human", "review": "distinct_reviewer", "summary": "A worker or person who neither produced the artifact, wrote a view nor wrote the synthesis checks every quoted position against its sealed view, looks for dropped points, disagreements sent the wrong way and head-count language, and passes the synthesis only when none is found.", "outcome": "A synthesis audit that passes only with nothing found.", "ready_outcome": "synthesis_audited", "pinned": [ "guardrail.disagreement-first", "rules.synthesis", "panel-checks" ], "slots": [ { "name": "synthesis", "category": "noun.seed_hypothesis", "description": "Pin of the synthesis." }, { "name": "views", "category": "noun.eval_run", "description": "Pins of the sealed views.", "many": true, "max": 7 }, { "name": "independence_check", "category": "noun.eval_run", "description": "Pin of the independence check." } ], "produces": [ { "name": "audit", "template": "template.synthesis-audit" } ], "steps": [ "Verify the pins and that you wrote none of the artifact, the views or the synthesis.", "Find every quote in its reviewer's sealed view, character for character.", "List every view point the synthesis dropped, every disagreement routed against the kind rules and every sentence that settles something by a count.", "Pass only with every list empty; propose a fresh synthesize-disagreements verb otherwise." ], "acceptance": [ { "id": "matches-template", "check": "The audit validates against template.synthesis-audit's schema.", "method": "deterministic" }, { "id": "quotes-rechecked", "check": "positions_checked equals the number of positions in the synthesis.", "method": "deterministic" }, { "id": "passes-only-clean", "check": "passed is true only when every list in the audit is empty.", "method": "deterministic" }, { "id": "different-auditor", "check": "The auditing key differs from the producer's, every reviewer's and the synthesizer's.", "method": "deterministic" } ], "bounds": { "max_elapsed_seconds": 1800 }, "follows": [ "persona-panel-second-opinion.synthesize-disagreements" ] }, { "slug": "decide", "kind": "verb.verify_gate", "title": "Decide, having read the disagreements (a person)", "claimant": "human", "model_use": "none", "summary": "A person reads the synthesis and its audit and decides: proceed, proceed with changes, do not proceed or decide later, saying how each disagreement was settled or what comes next for each one left open.", "outcome": "A person's decision on the synthesis.", "ready_outcome": "decision_made", "pinned": [ "policy-and-law-watch:guardrail.person-decides", "checklist" ], "slots": [ { "name": "synthesis", "category": "noun.seed_hypothesis", "description": "Pin of the synthesis." }, { "name": "audit", "category": "noun.eval_run", "description": "Pin of the synthesis audit." } ], "produces": [ { "name": "decision", "template": "template.decision" } ], "steps": [ "Run the checklist, reading the disagreements first.", "Settle each disagreement or leave it open with its next step; settle one of taste through the taste palettes pack's record-choice verb. Deciding later is valid." ], "acceptance": [ { "id": "matches-template", "check": "The decision validates against template.decision's schema.", "method": "deterministic" }, { "id": "every-disagreement-answered", "check": "Every disagreement id in the synthesis is in settled or left_open.", "method": "deterministic" }, { "id": "person-decides", "check": "The submitting key belongs to a person who neither produced the artifact, reviewed it nor wrote the synthesis.", "method": "deterministic" } ], "bounds": { "max_elapsed_seconds": 604800 }, "follows": [ "persona-panel-second-opinion.audit-synthesis" ] } ] }