{"format": "ta-amps.seed-pack.nouns.v1", "pack_id": "evaluation-design", "status": "proposal_only", "records": [
{"store": "seedpack.evaluation-design", "key": "evaluation-design.v0.1.0.pack", "expected_version": 0, "producer_surface": "surface:tessera", "visibility_scope": "surface_only", "allowed_surfaces": [], "target_cohorts": ["all"], "tags": ["category:noun.skill_pack", "pack:evaluation-design", "role:pack", "content:authored", "spdx:MIT", "status:quarantined"], "value": {"category_id": "noun.skill_pack", "role": "pack", "pack": "evaluation-design", "pack_version": "0.1.0", "title": "A collection of nouns and verbs for fair, pre-registered evaluations", "lets_workers": "Run a fair, honest comparison: fixed arms, a frozen case set with a manifest, pass marks a person commits before any run, exact-match scoring, two runs per arm, cost per case, and missed marks reported as missed.", "flow": [{"verb": "evaluation-design.design-evaluation", "when": "First: question, arms, metrics and draft pass marks."}, {"verb": "evaluation-design.freeze-set", "when": "Before any arm runs."}, {"verb": "evaluation-design.commit-pass-marks", "when": "A person approves the marks before the first scored run."}, {"verb": "evaluation-design.run-arm", "when": "Twice per arm, as separate attempts."}, {"verb": "evaluation-design.score-run", "when": "After each run; deterministic."}, {"verb": "evaluation-design.report-evaluation", "when": "After every run is scored."}, {"verb": "evaluation-design.review-evaluation", "when": "Last, by a different worker."}], "method": {"store": "seedpack.evaluation-design", "key": "evaluation-design.v0.1.0.method", "version": 1, "value_sha256": "436329f710d068a71b3e6f7252a8f4ff875bda0d6eac92424526cb5712be11db"}, "worked_example": {"store": "seedpack.evaluation-design", "key": "evaluation-design.v0.1.0.example.worked", "version": 1, "value_sha256": "68a2a02056104e275efaf2ea42233bf1276c33c9c9d4b18d5429fa7f1c859849"}, "used_by": ["tool-discovery trials", "decomposition-patterns comparisons", "any measured comparison in another pack"], "provenance_notes": ["The method generalizes docs/evaluation.md in the shamash repository (arms, frozen set, pass marks committed before runs, exact-match scoring, two runs per arm)."], "status": "proposal", "admission": "not_admitted", "content_kind": "authored_guidance", "license": "MIT", "provenance": {"source": "original ta-amps authored proposal", "license": "MIT", "generator": "scripts/build_packs.py", "independently_verified": false}, "dimensions": {"determinism": 0.0, "evidence_grounding": 0.0, "token_efficiency": 0.0, "safety_compliance": 0.0, "verification_tier": "unverified"}, "dimension_basis": {"measured": false, "method": "unmeasured_floor", "reason": "Tessera requires these axes, so each is set to its lowest value and the tier is unverified. No quality is claimed. Replace them only with measured values and the evidence that produced them."}, "relations": [{"relation": "requires_noun", "target_id": "seedpack.evaluation-design/evaluation-design.v0.1.0.method", "target_category": "noun.skill_pack", "note": ""}, {"relation": "requires_noun", "target_id": "seedpack.evaluation-design/evaluation-design.v0.1.0.scoring.exact-match", "target_category": "noun.reflex_procedure", "note": ""}, {"relation": "requires_noun", "target_id": "seedpack.evaluation-design/evaluation-design.v0.1.0.example.worked", "target_category": "noun.benchmark_suite", "note": ""}, {"relation": "requires_noun", "target_id": "seedpack.evaluation-design/evaluation-design.v0.1.0.template.evaluation-plan", "target_category": "noun.benchmark_suite", "note": ""}, {"relation": "requires_noun", "target_id": "seedpack.evaluation-design/evaluation-design.v0.1.0.template.arm", "target_category": "noun.skill_pack", "note": ""}, {"relation": "requires_noun", "target_id": "seedpack.evaluation-design/evaluation-design.v0.1.0.template.frozen-set-manifest", "target_category": "noun.benchmark_suite", "note": ""}, {"relation": "requires_noun", "target_id": "seedpack.evaluation-design/evaluation-design.v0.1.0.template.pass-marks", "target_category": "noun.seed_hypothesis", "note": ""}, {"relation": "requires_noun", "target_id": "seedpack.evaluation-design/evaluation-design.v0.1.0.template.run-record", "target_category": "noun.eval_run", "note": ""}, {"relation": "requires_noun", "target_id": "seedpack.evaluation-design/evaluation-design.v0.1.0.template.evaluation-report", "target_category": "noun.eval_run", "note": ""}]}},
{"store": "seedpack.evaluation-design", "key": "evaluation-design.v0.1.0.method", "expected_version": 0, "producer_surface": "surface:tessera", "visibility_scope": "surface_only", "allowed_surfaces": [], "target_cohorts": ["all"], "tags": ["category:noun.skill_pack", "pack:evaluation-design", "role:method", "content:authored", "spdx:MIT", "status:quarantined"], "value": {"category_id": "noun.skill_pack", "role": "method", "pack": "evaluation-design", "pack_version": "0.1.0", "title": "Evaluation method: arms, frozen sets, pass marks before runs, exact-match scoring", "category_reason": "A reusable evaluation method with guardrails, generalized from an evaluation plan used in this ecosystem.", "origin": "Generalizes an existing evaluation plan: re-run fixed arms against a frozen set, score by exact match, run each arm twice and commit pass marks first. The source is named in the descriptor's provenance_notes.", "steps": ["Write the question the evaluation answers in one sentence, and the decision it informs.", "Define arms: a baseline and each change, one change per arm where possible, with exact models, prompts, tools and settings.", "Freeze the case set before any arm runs: opaque case ids, a manifest of case and label digests, and the generator or source pinned.", "Commit pass marks before the first scored run. A person approves them; they never change afterwards.", "Run every arm twice on the full frozen set. Record cost and latency per case from actual usage.", "Score by exact match against known labels with deterministic code; a model judge may grade only explanations, with a person spot-checking its grades.", "Report every mark as met, missed or unknown. A missed mark is reported as missed with its diagnosis, never reworded into a pass."], "rules": ["Case ids must not encode labels; the drafting and answering workers never see labels.", "Errors and timeouts are scored as wrong answers unless the plan says to exclude them; exclusions are reported with counts.", "Cost per case is null when any case's cost is unknown; report the known subtotal and the count of unknown cases.", "Spread between the two runs of an arm above the plan's limit marks the arm unstable.", "Changing a pass mark, the set or the scorer after a run starts requires a new evaluation version; old results stay as they were."], "metrics_menu": {"accuracy_points": "Share of cases answered exactly right, in points.", "recall_points": "Share of positive cases answered with the positive label, in points.", "twin_fp_points": "Share of negative twin cases wrongly given the positive label, in points.", "net_points": "Recall minus twin false positives, in points.", "abstention_points": "Share of cases held for review instead of answered, when the plan allows abstaining.", "cost_per_case_usd": "Actual usage times dated price, divided by cases; null when any case cost is unknown.", "p95_latency_ms": "From recorded per-case timings."}, "status": "proposal", "admission": "not_admitted", "content_kind": "authored_guidance", "license": "MIT", "provenance": {"source": "original ta-amps authored proposal", "license": "MIT", "generator": "scripts/build_packs.py", "independently_verified": false}, "dimensions": {"determinism": 0.0, "evidence_grounding": 0.0, "token_efficiency": 0.0, "safety_compliance": 0.0, "verification_tier": "unverified"}, "dimension_basis": {"measured": false, "method": "unmeasured_floor", "reason": "Tessera requires these axes, so each is set to its lowest value and the tier is unverified. No quality is claimed. Replace them only with measured values and the evidence that produced them."}, "relations": []}},
{"store": "seedpack.evaluation-design", "key": "evaluation-design.v0.1.0.scoring.exact-match", "expected_version": 0, "producer_surface": "surface:tessera", "visibility_scope": "surface_only", "allowed_surfaces": [], "target_cohorts": ["all"], "tags": ["category:noun.reflex_procedure", "pack:evaluation-design", "role:scoring_procedure", "content:authored", "spdx:MIT", "status:quarantined"], "value": {"category_id": "noun.reflex_procedure", "role": "scoring_procedure", "pack": "evaluation-design", "pack_version": "0.1.0", "title": "Exact-match scoring procedure", "category_reason": "A zero-call deterministic procedure: the same predictions and labels always give the same scores.", "inputs": ["A run record", "The frozen set's labels, readable only by the scorer", "The committed pass marks"], "steps": ["Verify that the run's set reference and the manifest digests match.", "For each case, the prediction is correct only if it equals the label exactly; null or an error is wrong.", "Compute the plan's metrics in points with three decimals; cost per case is null if any case cost is null.", "Evaluate each pass mark across the arm's runs: met, missed, or unknown when a needed value is null."], "model_calls": 0, "checks": ["Scoring the same run twice gives identical output.", "The worked example's recorded results are reproduced exactly."], "worked_example": {"store": "seedpack.evaluation-design", "key": "evaluation-design.v0.1.0.example.worked", "version": 1, "value_sha256": "68a2a02056104e275efaf2ea42233bf1276c33c9c9d4b18d5429fa7f1c859849"}, "status": "proposal", "admission": "not_admitted", "content_kind": "authored_guidance", "license": "MIT", "provenance": {"source": "original ta-amps authored proposal", "license": "MIT", "generator": "scripts/build_packs.py", "independently_verified": false}, "dimensions": {"determinism": 0.0, "evidence_grounding": 0.0, "token_efficiency": 0.0, "safety_compliance": 0.0, "verification_tier": "unverified"}, "dimension_basis": {"measured": false, "method": "unmeasured_floor", "reason": "Tessera requires these axes, so each is set to its lowest value and the tier is unverified. No quality is claimed. Replace them only with measured values and the evidence that produced them."}, "relations": [{"relation": "requires_noun", "target_id": "seedpack.evaluation-design/evaluation-design.v0.1.0.method", "target_category": "noun.skill_pack", "note": ""}, {"relation": "verifies", "target_id": "seedpack.evaluation-design/evaluation-design.v0.1.0.example.worked", "target_category": "noun.benchmark_suite", "note": "reproduces the recorded expected results"}]}},
{"store": "seedpack.evaluation-design", "key": "evaluation-design.v0.1.0.example.worked", "expected_version": 0, "producer_surface": "surface:tessera", "visibility_scope": "surface_only", "allowed_surfaces": [], "target_cohorts": ["all"], "tags": ["category:noun.benchmark_suite", "pack:evaluation-design", "role:worked_example", "synthetic:true", "spdx:MIT", "status:quarantined"], "value": {"category_id": "noun.benchmark_suite", "role": "worked_example", "pack": "evaluation-design", "pack_version": "0.1.0", "title": "SYNTHETIC worked example: two arms, two runs each, four pass marks", "category_reason": "A golden fixture: fixed cases, fixed predictions and the recorded expected metrics and mark results.", "fixture_kind": "evaluation_arithmetic", "purpose": "Shows the method's arithmetic end to end, including an error counted as a miss, a missed mark and an unknown cost. The validator recomputes it.", "positive_label": "flag", "runs_per_arm": 2, "cases": [{"case_id": "case-01", "label": "flag"}, {"case_id": "case-02", "label": "clear"}, {"case_id": "case-03", "label": "flag"}, {"case_id": "case-04", "label": "clear"}, {"case_id": "case-05", "label": "clear"}, {"case_id": "case-06", "label": "flag"}, {"case_id": "case-07", "label": "clear"}, {"case_id": "case-08", "label": "flag"}], "runs": [{"run_id": "A0-1", "arm": "A0", "predictions": {"case-01": "flag", "case-02": "flag", "case-03": "clear", "case-04": "clear", "case-05": "clear", "case-06": "flag", "case-07": "clear", "case-08": "clear"}, "cost_usd": {"case-01": 0.002, "case-02": 0.002, "case-03": 0.002, "case-04": 0.002, "case-05": 0.002, "case-06": 0.002, "case-07": 0.002, "case-08": 0.002}}, {"run_id": "A0-2", "arm": "A0", "predictions": {"case-01": "flag", "case-02": "flag", "case-03": "flag", "case-04": "flag", "case-05": "clear", "case-06": "clear", "case-07": "clear", "case-08": "clear"}, "cost_usd": {"case-01": 0.002, "case-02": 0.002, "case-03": 0.002, "case-04": 0.002, "case-05": 0.002, "case-06": 0.002, "case-07": 0.002, "case-08": 0.002}}, {"run_id": "A1-1", "arm": "A1", "predictions": {"case-01": "flag", "case-02": "clear", "case-03": "flag", "case-04": "clear", "case-05": "flag", "case-06": "flag", "case-07": "clear", "case-08": "flag"}, "cost_usd": {"case-01": 0.01, "case-02": 0.01, "case-03": 0.01, "case-04": 0.01, "case-05": 0.01, "case-06": 0.01, "case-07": 0.01, "case-08": 0.01}}, {"run_id": "A1-2", "arm": "A1", "predictions": {"case-01": "flag", "case-02": "clear", "case-03": "flag", "case-04": "clear", "case-05": "clear", "case-06": "flag", "case-07": "clear", "case-08": null}, "cost_usd": {"case-01": 0.01, "case-02": 0.01, "case-03": 0.01, "case-04": 0.01, "case-05": 0.01, "case-06": 0.01, "case-07": 0.01, "case-08": null}}], "pass_marks": [{"id": "M1", "metric": "net_gain_points", "arm": "A1", "baseline": "A0", "threshold": 25, "rule": "A1's net points beat A0's by at least 25 in both runs."}, {"id": "M2", "metric": "twin_fp_not_higher", "arm": "A1", "baseline": "A0", "threshold": null, "rule": "A1's twin false positives are no higher than A0's in both runs."}, {"id": "M3", "metric": "recall_at_least", "arm": "A1", "threshold": 90, "rule": "A1's recall is at least 90 points in both runs."}, {"id": "M4", "metric": "spread_points_below", "arm": "A1", "threshold": 5, "rule": "A1's net points differ by less than 5 between its two runs."}, {"id": "M5", "metric": "cost_per_case_at_most", "arm": "A1", "threshold": 0.02, "rule": "A1 costs at most 0.02 dollars per case in both runs."}], "expected": {"metrics": {"A0-1": {"arm": "A0", "recall_points": 50.0, "twin_fp_points": 25.0, "net_points": 25.0, "errors": 0, "cost_per_case_usd": 0.002}, "A0-2": {"arm": "A0", "recall_points": 50.0, "twin_fp_points": 50.0, "net_points": 0.0, "errors": 0, "cost_per_case_usd": 0.002}, "A1-1": {"arm": "A1", "recall_points": 100.0, "twin_fp_points": 25.0, "net_points": 75.0, "errors": 0, "cost_per_case_usd": 0.01}, "A1-2": {"arm": "A1", "recall_points": 75.0, "twin_fp_points": 0.0, "net_points": 75.0, "errors": 1, "cost_per_case_usd": null}}, "marks": {"M1": "met", "M2": "met", "M3": "missed", "M4": "met", "M5": "unknown"}}, "reading": ["M3 is missed because the error on case-08 counts as a wrong answer; it is reported as missed, not excused.", "M5 is unknown because one case's cost was not recorded; the known subtotal is reported beside it."], "status": "proposal", "admission": "not_admitted", "content_kind": "synthetic_example", "license": "MIT", "provenance": {"source": "original ta-amps authored proposal", "license": "MIT", "generator": "scripts/build_packs.py", "independently_verified": false}, "dimensions": {"determinism": 0.0, "evidence_grounding": 0.0, "token_efficiency": 0.0, "safety_compliance": 0.0, "verification_tier": "unverified"}, "dimension_basis": {"measured": false, "method": "unmeasured_floor", "reason": "Tessera requires these axes, so each is set to its lowest value and the tier is unverified. No quality is claimed. Replace them only with measured values and the evidence that produced them."}, "relations": [{"relation": "requires_noun", "target_id": "seedpack.evaluation-design/evaluation-design.v0.1.0.method", "target_category": "noun.skill_pack", "note": ""}]}},
{"store": "seedpack.evaluation-design", "key": "evaluation-design.v0.1.0.template.evaluation-plan", "expected_version": 0, "producer_surface": "surface:tessera", "visibility_scope": "surface_only", "allowed_surfaces": [], "target_cohorts": ["all"], "tags": ["category:noun.benchmark_suite", "pack:evaluation-design", "role:template", "content:authored", "spdx:MIT", "status:quarantined"], "value": {"category_id": "noun.benchmark_suite", "role": "template", "pack": "evaluation-design", "pack_version": "0.1.0", "title": "Template: an evaluation plan: question, decision, arms, metrics, runs per arm, scoring and cost accounting", "category_reason": "The plan fixes the harness, metrics and scoring rubric before any run.", "describes": "an evaluation plan: question, decision, arms, metrics, runs per arm, scoring and cost accounting", "schema": {"type": "object", "additionalProperties": false, "required": ["question", "decision", "arms", "set_ref", "metrics", "runs_per_arm", "scoring", "spread_limit_points", "cost_accounting"], "properties": {"question": {"type": "string", "minLength": 1, "maxLength": 300}, "decision": {"type": "string", "minLength": 1, "maxLength": 300}, "arms": {"type": "array", "minItems": 2, "maxItems": 10, "items": {"type": "object", "additionalProperties": false, "required": ["arm_id", "role", "change"], "properties": {"arm_id": {"type": "string", "minLength": 1, "maxLength": 20, "pattern": "^[A-Za-z0-9_-]+$"}, "role": {"type": "string", "enum": ["baseline", "candidate", "control"]}, "change": {"type": "string", "minLength": 1, "maxLength": 300}, "arm_ref": {"type": "string", "minLength": 0, "maxLength": 300}}}}, "set_ref": {"type": "string", "minLength": 0, "maxLength": 300}, "metrics": {"type": "array", "minItems": 1, "maxItems": 10, "items": {"type": "string", "minLength": 1, "maxLength": 60}}, "runs_per_arm": {"type": "integer", "minimum": 2, "maximum": 5}, "scoring": {"type": "object", "additionalProperties": false, "required": ["method", "judge_use"], "properties": {"method": {"type": "string", "enum": ["exact_match"]}, "judge_use": {"type": "string", "enum": ["none", "explanations_only_with_human_spot_check"]}}}, "spread_limit_points": {"type": "number", "minimum": 0, "maximum": 100}, "cost_accounting": {"type": "string", "minLength": 1, "maxLength": 300}, "exclusions_rule": {"type": "string", "minLength": 0, "maxLength": 300}}}, "rules": ["At least one baseline arm.", "runs_per_arm is at least 2.", "Scoring is exact match; a judge, if any, grades explanations only."], "status": "proposal", "admission": "not_admitted", "content_kind": "authored_guidance", "license": "MIT", "provenance": {"source": "original ta-amps authored proposal", "license": "MIT", "generator": "scripts/build_packs.py", "independently_verified": false}, "dimensions": {"determinism": 0.0, "evidence_grounding": 0.0, "token_efficiency": 0.0, "safety_compliance": 0.0, "verification_tier": "unverified"}, "dimension_basis": {"measured": false, "method": "unmeasured_floor", "reason": "Tessera requires these axes, so each is set to its lowest value and the tier is unverified. No quality is claimed. Replace them only with measured values and the evidence that produced them."}, "relations": []}},
{"store": "seedpack.evaluation-design", "key": "evaluation-design.v0.1.0.template.arm", "expected_version": 0, "producer_surface": "surface:tessera", "visibility_scope": "surface_only", "allowed_surfaces": [], "target_cohorts": ["all"], "tags": ["category:noun.skill_pack", "pack:evaluation-design", "role:template", "content:authored", "spdx:MIT", "status:quarantined"], "value": {"category_id": "noun.skill_pack", "role": "template", "pack": "evaluation-design", "pack_version": "0.1.0", "title": "Template: one arm: exact provider, model, prompt, tools, context, settings and budget", "category_reason": "An arm is a reusable agent configuration: model, prompt, tool bindings and guardrails.", "describes": "one arm: exact provider, model, prompt, tools, context, settings and budget", "schema": {"type": "object", "additionalProperties": false, "required": ["arm_id", "provider", "model", "prompt_ref", "tools", "context", "settings", "budget"], "properties": {"arm_id": {"type": "string", "minLength": 1, "maxLength": 20, "pattern": "^[A-Za-z0-9_-]+$"}, "provider": {"type": "string", "minLength": 1, "maxLength": 100}, "model": {"type": "string", "minLength": 1, "maxLength": 150}, "prompt_ref": {"type": "string", "minLength": 0, "maxLength": 300}, "tools": {"type": "array", "minItems": 0, "maxItems": 20, "items": {"type": "string", "minLength": 1, "maxLength": 150}}, "context": {"type": "string", "minLength": 1, "maxLength": 400}, "settings": {"type": "object", "additionalProperties": false, "required": ["source"], "properties": {"source": {"type": "string", "enum": ["caller_defaults", "explicit"]}, "explicit": {"type": "string", "minLength": 0, "maxLength": 400}}}, "budget": {"type": "object", "additionalProperties": false, "required": ["max_usd", "max_calls"], "properties": {"max_usd": {"type": "number", "minimum": 0, "maximum": 1000}, "max_calls": {"type": "integer", "minimum": 0}}}}}, "rules": ["Name the exact model identifier.", "settings.source is caller_defaults unless the plan changes a setting on purpose."], "status": "proposal", "admission": "not_admitted", "content_kind": "authored_guidance", "license": "MIT", "provenance": {"source": "original ta-amps authored proposal", "license": "MIT", "generator": "scripts/build_packs.py", "independently_verified": false}, "dimensions": {"determinism": 0.0, "evidence_grounding": 0.0, "token_efficiency": 0.0, "safety_compliance": 0.0, "verification_tier": "unverified"}, "dimension_basis": {"measured": false, "method": "unmeasured_floor", "reason": "Tessera requires these axes, so each is set to its lowest value and the tier is unverified. No quality is claimed. Replace them only with measured values and the evidence that produced them."}, "relations": []}},
{"store": "seedpack.evaluation-design", "key": "evaluation-design.v0.1.0.template.frozen-set-manifest", "expected_version": 0, "producer_surface": "surface:tessera", "visibility_scope": "surface_only", "allowed_surfaces": [], "target_cohorts": ["all"], "tags": ["category:noun.benchmark_suite", "pack:evaluation-design", "role:template", "content:authored", "spdx:MIT", "status:quarantined"], "value": {"category_id": "noun.benchmark_suite", "role": "template", "pack": "evaluation-design", "pack_version": "0.1.0", "title": "Template: the frozen case set's manifest: opaque ids, case and label digests, splits and groups", "category_reason": "A frozen-set manifest is a fixture manifest with anti-contamination digests.", "describes": "the frozen case set's manifest: opaque ids, case and label digests, splits and groups", "schema": {"type": "object", "additionalProperties": false, "required": ["set_id", "frozen_at", "generator_ref", "cases", "label_access"], "properties": {"set_id": {"type": "string", "minLength": 1, "maxLength": 80}, "frozen_at": {"type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}", "maxLength": 40}, "generator_ref": {"type": "string", "minLength": 1, "maxLength": 300}, "cases": {"type": "array", "minItems": 2, "maxItems": 2000, "items": {"type": "object", "additionalProperties": false, "required": ["case_id", "input_sha256", "label_sha256", "split", "group"], "properties": {"case_id": {"type": "string", "minLength": 1, "maxLength": 40, "pattern": "^[a-z0-9-]+$"}, "input_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, "label_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, "split": {"type": "string", "enum": ["dev", "holdout", "adversarial"]}, "group": {"type": "string", "minLength": 1, "maxLength": 60}}}}, "label_access": {"type": "string", "enum": ["scorer_only"]}, "id_check": {"type": "string", "minLength": 0, "maxLength": 300}}}, "rules": ["Case ids are opaque and never encode labels.", "Labels are readable only by the scorer.", "Near-duplicates share a group and a split."], "status": "proposal", "admission": "not_admitted", "content_kind": "authored_guidance", "license": "MIT", "provenance": {"source": "original ta-amps authored proposal", "license": "MIT", "generator": "scripts/build_packs.py", "independently_verified": false}, "dimensions": {"determinism": 0.0, "evidence_grounding": 0.0, "token_efficiency": 0.0, "safety_compliance": 0.0, "verification_tier": "unverified"}, "dimension_basis": {"measured": false, "method": "unmeasured_floor", "reason": "Tessera requires these axes, so each is set to its lowest value and the tier is unverified. No quality is claimed. Replace them only with measured values and the evidence that produced them."}, "relations": []}},
{"store": "seedpack.evaluation-design", "key": "evaluation-design.v0.1.0.template.pass-marks", "expected_version": 0, "producer_surface": "surface:tessera", "visibility_scope": "surface_only", "allowed_surfaces": [], "target_cohorts": ["all"], "tags": ["category:noun.seed_hypothesis", "pack:evaluation-design", "role:template", "content:authored", "spdx:MIT", "status:quarantined"], "value": {"category_id": "noun.seed_hypothesis", "role": "template", "pack": "evaluation-design", "pack_version": "0.1.0", "title": "Template: pass marks a person commits before the first scored run", "category_reason": "Pass marks are pre-registered falsifiable predictions about the arms.", "describes": "pass marks a person commits before the first scored run", "schema": {"type": "object", "additionalProperties": false, "required": ["evaluation_ref", "marks", "committed_by", "committed_at", "immutable"], "properties": {"evaluation_ref": {"type": "string", "minLength": 1, "maxLength": 300}, "marks": {"type": "array", "minItems": 1, "maxItems": 12, "items": {"type": "object", "additionalProperties": false, "required": ["id", "metric", "arm", "rule"], "properties": {"id": {"type": "string", "minLength": 1, "maxLength": 20}, "metric": {"type": "string", "minLength": 1, "maxLength": 60}, "arm": {"type": "string", "minLength": 1, "maxLength": 20}, "baseline": {"type": "string", "minLength": 0, "maxLength": 20}, "threshold": {"type": ["number", "null"]}, "rule": {"type": "string", "minLength": 1, "maxLength": 300}}}}, "committed_by": {"type": "string", "enum": ["human"]}, "committed_at": {"type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}", "maxLength": 40}, "immutable": {"type": "boolean", "enum": [true]}}}, "rules": ["committed_by is human.", "immutable is true; a change needs a new evaluation version."], "status": "proposal", "admission": "not_admitted", "content_kind": "authored_guidance", "license": "MIT", "provenance": {"source": "original ta-amps authored proposal", "license": "MIT", "generator": "scripts/build_packs.py", "independently_verified": false}, "dimensions": {"determinism": 0.0, "evidence_grounding": 0.0, "token_efficiency": 0.0, "safety_compliance": 0.0, "verification_tier": "unverified"}, "dimension_basis": {"measured": false, "method": "unmeasured_floor", "reason": "Tessera requires these axes, so each is set to its lowest value and the tier is unverified. No quality is claimed. Replace them only with measured values and the evidence that produced them."}, "relations": []}},
{"store": "seedpack.evaluation-design", "key": "evaluation-design.v0.1.0.template.run-record", "expected_version": 0, "producer_surface": "surface:tessera", "visibility_scope": "surface_only", "allowed_surfaces": [], "target_cohorts": ["all"], "tags": ["category:noun.eval_run", "pack:evaluation-design", "role:template", "content:authored", "spdx:MIT", "status:quarantined"], "value": {"category_id": "noun.eval_run", "role": "template", "pack": "evaluation-design", "pack_version": "0.1.0", "title": "Template: one run of one arm: per-case predictions, cost and latency, and usage", "category_reason": "A run record is a fingerprinted run with null-honest per-case measurements.", "describes": "one run of one arm: per-case predictions, cost and latency, and usage", "schema": {"type": "object", "additionalProperties": false, "required": ["arm_id", "run_index", "set_ref", "marks_ref", "cases", "usage"], "properties": {"arm_id": {"type": "string", "minLength": 1, "maxLength": 20}, "run_index": {"type": "integer", "minimum": 1, "maximum": 5}, "set_ref": {"type": "string", "minLength": 1, "maxLength": 300}, "marks_ref": {"type": "string", "minLength": 1, "maxLength": 300}, "cases": {"type": "array", "minItems": 1, "maxItems": 2000, "items": {"type": "object", "additionalProperties": false, "required": ["case_id", "prediction", "cost_usd", "latency_ms"], "properties": {"case_id": {"type": "string", "minLength": 1, "maxLength": 40}, "prediction": {"type": ["string", "null"]}, "error": {"type": "string", "minLength": 0, "maxLength": 300}, "cost_usd": {"type": ["number", "null"]}, "latency_ms": {"type": ["number", "null"]}}}}, "usage": {"type": "object", "additionalProperties": false, "required": ["source"], "properties": {"source": {"type": "string", "minLength": 1, "maxLength": 120}, "input_tokens": {"type": ["integer", "null"]}, "output_tokens": {"type": ["integer", "null"]}, "reason": {"type": "string", "minLength": 0, "maxLength": 300}}}}}, "rules": ["One record per arm per run.", "Unknown cost or tokens stay null with a reason."], "status": "proposal", "admission": "not_admitted", "content_kind": "authored_guidance", "license": "MIT", "provenance": {"source": "original ta-amps authored proposal", "license": "MIT", "generator": "scripts/build_packs.py", "independently_verified": false}, "dimensions": {"determinism": 0.0, "evidence_grounding": 0.0, "token_efficiency": 0.0, "safety_compliance": 0.0, "verification_tier": "unverified"}, "dimension_basis": {"measured": false, "method": "unmeasured_floor", "reason": "Tessera requires these axes, so each is set to its lowest value and the tier is unverified. No quality is claimed. Replace them only with measured values and the evidence that produced them."}, "relations": []}},
{"store": "seedpack.evaluation-design", "key": "evaluation-design.v0.1.0.template.evaluation-report", "expected_version": 0, "producer_surface": "surface:tessera", "visibility_scope": "surface_only", "allowed_surfaces": [], "target_cohorts": ["all"], "tags": ["category:noun.eval_run", "pack:evaluation-design", "role:template", "content:authored", "spdx:MIT", "status:quarantined"], "value": {"category_id": "noun.eval_run", "role": "template", "pack": "evaluation-design", "pack_version": "0.1.0", "title": "Template: the evaluation report: every mark met, missed or unknown, per-arm metrics, stability and limits", "category_reason": "The report is the scorecard across arms and runs.", "describes": "the evaluation report: every mark met, missed or unknown, per-arm metrics, stability and limits", "schema": {"type": "object", "additionalProperties": false, "required": ["evaluation_ref", "marks", "arms", "limits"], "properties": {"evaluation_ref": {"type": "string", "minLength": 1, "maxLength": 300}, "marks": {"type": "array", "minItems": 1, "maxItems": 12, "items": {"type": "object", "additionalProperties": false, "required": ["id", "result", "detail"], "properties": {"id": {"type": "string", "minLength": 1, "maxLength": 20}, "result": {"type": "string", "enum": ["met", "missed", "unknown"]}, "detail": {"type": "string", "minLength": 1, "maxLength": 400}, "diagnosis": {"type": "string", "minLength": 0, "maxLength": 600}}}}, "arms": {"type": "array", "minItems": 1, "maxItems": 10, "items": {"type": "object", "additionalProperties": false, "required": ["arm_id", "runs", "stable"], "properties": {"arm_id": {"type": "string", "minLength": 1, "maxLength": 20}, "stable": {"type": "boolean"}, "runs": {"type": "array", "minItems": 1, "maxItems": 5, "items": {"type": "object", "additionalProperties": false, "required": ["run_index", "metrics"], "properties": {"run_index": {"type": "integer", "minimum": 1, "maximum": 5}, "metrics": {"type": "array", "minItems": 0, "maxItems": 12, "items": {"type": "object", "additionalProperties": false, "required": ["name", "value"], "properties": {"name": {"type": "string", "minLength": 1, "maxLength": 60}, "value": {"type": ["number", "null"]}}}}}}}}}}, "limits": {"type": "array", "minItems": 1, "maxItems": 10, "items": {"type": "string", "minLength": 1, "maxLength": 300}}}}, "rules": ["Every committed mark appears.", "A missed mark has a diagnosis.", "Limits list what the set and arms do not cover."], "status": "proposal", "admission": "not_admitted", "content_kind": "authored_guidance", "license": "MIT", "provenance": {"source": "original ta-amps authored proposal", "license": "MIT", "generator": "scripts/build_packs.py", "independently_verified": false}, "dimensions": {"determinism": 0.0, "evidence_grounding": 0.0, "token_efficiency": 0.0, "safety_compliance": 0.0, "verification_tier": "unverified"}, "dimension_basis": {"measured": false, "method": "unmeasured_floor", "reason": "Tessera requires these axes, so each is set to its lowest value and the tier is unverified. No quality is claimed. Replace them only with measured values and the evidence that produced them."}, "relations": []}}
]}