{ "pack_id": "repo-review", "title": "Repository review", "version": "0.1.0", "summary": "A collection of nouns and verbs for reviewing a code repository with evidence: fifty questions in five families (cost, reliability and security, speed, reach and distribution, revenue), with verbs to profile, select questions, investigate, compare fixes, report privately and review independently.", "dependencies": [], "boundaries": [ "Findings stay inside the requester's scope; nothing is published.", "No verb contacts the reviewed project's maintainers, users or services.", "Repository code runs only when the requester allows it, in an isolated sandbox without credentials.", "Importing the pack writes nothing and enqueues nothing." ], "nouns": [ { "slug": "pack", "category": "noun.skill_pack", "role": "pack", "title": "A collection of nouns and verbs for reviewing a code repository with evidence", "value": { "lets_workers": "Review a code repository with evidence: profile it, choose the questions that matter, investigate them, compare fixes and write a short report that an independent reviewer checks.", "question_bank": { "$pin": "question-bank" }, "evidence_standard": { "$pin": "evidence-standard" }, "flow": [ { "verb": { "$verb": "repo-review.profile-repo" }, "when": "First, once per commit." }, { "verb": { "$verb": "repo-review.select-questions" }, "when": "After the profile exists." }, { "verb": { "$verb": "repo-review.investigate-family" }, "when": "Once per family that has selected questions; families can run in parallel as separate attempts." }, { "verb": { "$verb": "repo-review.compare-options" }, "when": "Optional, for a confirmed concern with more than one plausible fix." }, { "verb": { "$verb": "repo-review.synthesize-report" }, "when": "After the family investigations finish." }, { "verb": { "$verb": "repo-review.independent-review" }, "when": "Last, by a different worker than the report's author." } ], "privacy": "Findings stay inside the requester's scope. No verb contacts the reviewed project's maintainers, users or services, and nothing is published.", "applies_to": "Any repository the requester is permitted to read, public or private." }, "relations": [ [ "requires_noun", "question-bank" ], [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.cost" ], [ "requires_noun", "family.reliability-security" ], [ "requires_noun", "family.speed" ], [ "requires_noun", "family.reach-distribution" ], [ "requires_noun", "family.revenue" ], [ "requires_noun", "template.repo-snapshot" ], [ "requires_noun", "template.repo-profile" ], [ "requires_noun", "template.review-plan" ], [ "requires_noun", "template.finding" ], [ "requires_noun", "template.option-comparison" ], [ "requires_noun", "template.review-report" ], [ "requires_noun", "template.review-verdict" ] ] }, { "slug": "question-bank", "category": "noun.benchmark_suite", "role": "question_bank", "title": "Repository review question bank: 50 questions in five families", "category_reason": "A fixed, versioned set of questions with rubrics that every review is judged against works like an evaluation suite for repositories.", "value": { "purpose": "Sharp questions that each require evidence. A review selects the few that could change a decision for this repository.", "families": [ { "id": "cost", "title": "Cost", "guide": { "$pin": "family.cost" }, "questions": 10 }, { "id": "reliability-security", "title": "Reliability and security", "guide": { "$pin": "family.reliability-security" }, "questions": 12 }, { "id": "speed", "title": "Speed", "guide": { "$pin": "family.speed" }, "questions": 10 }, { "id": "reach-distribution", "title": "Reach and distribution", "guide": { "$pin": "family.reach-distribution" }, "questions": 10 }, { "id": "revenue", "title": "Revenue", "guide": { "$pin": "family.revenue" }, "questions": 8 } ], "questions": [ { "id": "cost.idle-compute", "family": "cost", "pin": { "$pin": "question.cost.idle-compute" } }, { "id": "cost.ci-minutes", "family": "cost", "pin": { "$pin": "question.cost.ci-minutes" } }, { "id": "cost.dependency-caching", "family": "cost", "pin": { "$pin": "question.cost.dependency-caching" } }, { "id": "cost.model-spend", "family": "cost", "pin": { "$pin": "question.cost.model-spend" } }, { "id": "cost.retry-amplification", "family": "cost", "pin": { "$pin": "question.cost.retry-amplification" } }, { "id": "cost.storage-growth", "family": "cost", "pin": { "$pin": "question.cost.storage-growth" } }, { "id": "cost.per-request-operations", "family": "cost", "pin": { "$pin": "question.cost.per-request-operations" } }, { "id": "cost.plan-limit-cliffs", "family": "cost", "pin": { "$pin": "question.cost.plan-limit-cliffs" } }, { "id": "cost.unused-services", "family": "cost", "pin": { "$pin": "question.cost.unused-services" } }, { "id": "cost.artifact-size", "family": "cost", "pin": { "$pin": "question.cost.artifact-size" } }, { "id": "reliability-security.secret-exposure", "family": "reliability-security", "pin": { "$pin": "question.reliability-security.secret-exposure" } }, { "id": "reliability-security.authorization", "family": "reliability-security", "pin": { "$pin": "question.reliability-security.authorization" } }, { "id": "reliability-security.input-validation", "family": "reliability-security", "pin": { "$pin": "question.reliability-security.input-validation" } }, { "id": "reliability-security.idempotency", "family": "reliability-security", "pin": { "$pin": "question.reliability-security.idempotency" } }, { "id": "reliability-security.dependency-health", "family": "reliability-security", "pin": { "$pin": "question.reliability-security.dependency-health" } }, { "id": "reliability-security.dependency-failure", "family": "reliability-security", "pin": { "$pin": "question.reliability-security.dependency-failure" } }, { "id": "reliability-security.critical-path-tests", "family": "reliability-security", "pin": { "$pin": "question.reliability-security.critical-path-tests" } }, { "id": "reliability-security.backup-restore", "family": "reliability-security", "pin": { "$pin": "question.reliability-security.backup-restore" } }, { "id": "reliability-security.rollback", "family": "reliability-security", "pin": { "$pin": "question.reliability-security.rollback" } }, { "id": "reliability-security.observability", "family": "reliability-security", "pin": { "$pin": "question.reliability-security.observability" } }, { "id": "reliability-security.ci-supply-chain", "family": "reliability-security", "pin": { "$pin": "question.reliability-security.ci-supply-chain" } }, { "id": "reliability-security.personal-data", "family": "reliability-security", "pin": { "$pin": "question.reliability-security.personal-data" } }, { "id": "speed.first-success", "family": "speed", "pin": { "$pin": "question.speed.first-success" } }, { "id": "speed.ci-duration", "family": "speed", "pin": { "$pin": "question.speed.ci-duration" } }, { "id": "speed.test-suite", "family": "speed", "pin": { "$pin": "question.speed.test-suite" } }, { "id": "speed.hot-path-latency", "family": "speed", "pin": { "$pin": "question.speed.hot-path-latency" } }, { "id": "speed.cold-start", "family": "speed", "pin": { "$pin": "question.speed.cold-start" } }, { "id": "speed.n-plus-one", "family": "speed", "pin": { "$pin": "question.speed.n-plus-one" } }, { "id": "speed.page-weight", "family": "speed", "pin": { "$pin": "question.speed.page-weight" } }, { "id": "speed.release-path", "family": "speed", "pin": { "$pin": "question.speed.release-path" } }, { "id": "speed.review-latency", "family": "speed", "pin": { "$pin": "question.speed.review-latency" } }, { "id": "speed.dev-loop", "family": "speed", "pin": { "$pin": "question.speed.dev-loop" } }, { "id": "reach-distribution.first-screen", "family": "reach-distribution", "pin": { "$pin": "question.reach-distribution.first-screen" } }, { "id": "reach-distribution.install-paths", "family": "reach-distribution", "pin": { "$pin": "question.reach-distribution.install-paths" } }, { "id": "reach-distribution.discoverability", "family": "reach-distribution", "pin": { "$pin": "question.reach-distribution.discoverability" } }, { "id": "reach-distribution.docs-coverage", "family": "reach-distribution", "pin": { "$pin": "question.reach-distribution.docs-coverage" } }, { "id": "reach-distribution.integrations", "family": "reach-distribution", "pin": { "$pin": "question.reach-distribution.integrations" } }, { "id": "reach-distribution.license-fit", "family": "reach-distribution", "pin": { "$pin": "question.reach-distribution.license-fit" } }, { "id": "reach-distribution.accessibility", "family": "reach-distribution", "pin": { "$pin": "question.reach-distribution.accessibility" } }, { "id": "reach-distribution.demand-signals", "family": "reach-distribution", "pin": { "$pin": "question.reach-distribution.demand-signals" } }, { "id": "reach-distribution.demo-truth", "family": "reach-distribution", "pin": { "$pin": "question.reach-distribution.demo-truth" } }, { "id": "reach-distribution.onboarding-dropoff", "family": "reach-distribution", "pin": { "$pin": "question.reach-distribution.onboarding-dropoff" } }, { "id": "revenue.who-pays", "family": "revenue", "pin": { "$pin": "question.revenue.who-pays" } }, { "id": "revenue.pricing-unit", "family": "revenue", "pin": { "$pin": "question.revenue.pricing-unit" } }, { "id": "revenue.unit-cost", "family": "revenue", "pin": { "$pin": "question.revenue.unit-cost" } }, { "id": "revenue.free-paid-boundary", "family": "revenue", "pin": { "$pin": "question.revenue.free-paid-boundary" } }, { "id": "revenue.payment-safety", "family": "revenue", "pin": { "$pin": "question.revenue.payment-safety" } }, { "id": "revenue.funding-programs", "family": "revenue", "pin": { "$pin": "question.revenue.funding-programs" } }, { "id": "revenue.adoption-requirements", "family": "revenue", "pin": { "$pin": "question.revenue.adoption-requirements" } }, { "id": "revenue.heavy-user-risk", "family": "revenue", "pin": { "$pin": "question.revenue.heavy-user-risk" } } ], "selection_rules": [ "Select a question only when its applies_when conditions hold for the profile and an answer could change what the requester does next.", "Prefer questions whose evidence is reachable with the permitted access; mark the rest skipped with the reason.", "Select at most the requester's max_questions, ranked by expected decision value, with priority 1 for the most consequential.", "Never add questions outside the bank in this pack version; propose them as candidate nouns instead." ], "rubric_scale": { "strong": "Meets every strong_answer criterion of the question and every deterministic check of the evidence standard.", "adequate": "Answers with located evidence but misses one strong_answer criterion and says which.", "weak": "Evidence is partial or indirect and the finding does not say what is missing.", "reject": "Meets any fails_if condition, invents a number, cites nothing or fails a deterministic check." } }, "relations": [ [ "requires_noun", "question.cost.idle-compute" ], [ "requires_noun", "question.cost.ci-minutes" ], [ "requires_noun", "question.cost.dependency-caching" ], [ "requires_noun", "question.cost.model-spend" ], [ "requires_noun", "question.cost.retry-amplification" ], [ "requires_noun", "question.cost.storage-growth" ], [ "requires_noun", "question.cost.per-request-operations" ], [ "requires_noun", "question.cost.plan-limit-cliffs" ], [ "requires_noun", "question.cost.unused-services" ], [ "requires_noun", "question.cost.artifact-size" ], [ "requires_noun", "question.reliability-security.secret-exposure" ], [ "requires_noun", "question.reliability-security.authorization" ], [ "requires_noun", "question.reliability-security.input-validation" ], [ "requires_noun", "question.reliability-security.idempotency" ], [ "requires_noun", "question.reliability-security.dependency-health" ], [ "requires_noun", "question.reliability-security.dependency-failure" ], [ "requires_noun", "question.reliability-security.critical-path-tests" ], [ "requires_noun", "question.reliability-security.backup-restore" ], [ "requires_noun", "question.reliability-security.rollback" ], [ "requires_noun", "question.reliability-security.observability" ], [ "requires_noun", "question.reliability-security.ci-supply-chain" ], [ "requires_noun", "question.reliability-security.personal-data" ], [ "requires_noun", "question.speed.first-success" ], [ "requires_noun", "question.speed.ci-duration" ], [ "requires_noun", "question.speed.test-suite" ], [ "requires_noun", "question.speed.hot-path-latency" ], [ "requires_noun", "question.speed.cold-start" ], [ "requires_noun", "question.speed.n-plus-one" ], [ "requires_noun", "question.speed.page-weight" ], [ "requires_noun", "question.speed.release-path" ], [ "requires_noun", "question.speed.review-latency" ], [ "requires_noun", "question.speed.dev-loop" ], [ "requires_noun", "question.reach-distribution.first-screen" ], [ "requires_noun", "question.reach-distribution.install-paths" ], [ "requires_noun", "question.reach-distribution.discoverability" ], [ "requires_noun", "question.reach-distribution.docs-coverage" ], [ "requires_noun", "question.reach-distribution.integrations" ], [ "requires_noun", "question.reach-distribution.license-fit" ], [ "requires_noun", "question.reach-distribution.accessibility" ], [ "requires_noun", "question.reach-distribution.demand-signals" ], [ "requires_noun", "question.reach-distribution.demo-truth" ], [ "requires_noun", "question.reach-distribution.onboarding-dropoff" ], [ "requires_noun", "question.revenue.who-pays" ], [ "requires_noun", "question.revenue.pricing-unit" ], [ "requires_noun", "question.revenue.unit-cost" ], [ "requires_noun", "question.revenue.free-paid-boundary" ], [ "requires_noun", "question.revenue.payment-safety" ], [ "requires_noun", "question.revenue.funding-programs" ], [ "requires_noun", "question.revenue.adoption-requirements" ], [ "requires_noun", "question.revenue.heavy-user-risk" ], [ "requires_noun", "evidence-standard" ] ] }, { "slug": "family.cost", "category": "noun.skill_pack", "role": "family_guide", "title": "Investigation guide: cost", "category_reason": "A family guide is a reusable investigation method with its guardrails, used by every worker that investigates this family.", "value": { "family": "cost", "scope": "Money the project spends to build, test, host and serve: compute, CI minutes, storage, requests, paid APIs, model calls and subscriptions.", "look_at": [ "Deployment and infrastructure configuration.", "CI workflow files and recent run history.", "Code that calls paid APIs or models.", "Scheduled jobs, polling loops and retention settings.", "Provider pricing pages, recorded with the date read." ], "cautions": [ "Keep measured spend, list-price estimates and credits separate.", "Use the usage numbers the requester supplied; never invent traffic.", "A saving that weakens reliability, security or the caller's chosen model settings is a trade-off to report, not a free win." ], "out_of_scope": [ "Opening billing dashboards or credential files.", "Changing plans, settings or resources." ], "privacy": "Findings go only to the requester. Never contact the project's maintainers, users or services." }, "relations": [ [ "requires_noun", "evidence-standard" ] ] }, { "slug": "family.reliability-security", "category": "noun.skill_pack", "role": "family_guide", "title": "Investigation guide: reliability and security", "category_reason": "A family guide is a reusable investigation method with its guardrails, used by every worker that investigates this family.", "value": { "family": "reliability-security", "scope": "Whether the project keeps working and keeps data and credentials safe: authorization, input handling, secrets, dependencies, failure handling, tests, backups, rollback and monitoring.", "look_at": [ "Route handlers and authorization code.", "Validation code and size limits.", "CI workflows and their permissions.", "Lockfiles and offline advisory results.", "Retry, timeout and failure-handling code.", "Test suites, deploy scripts and backup scripts." ], "cautions": [ "Read code and run local tests only. Never probe, scan or load-test a live deployment, and never try a found credential.", "Mask any secret you see; report its location, not its value.", "Every advisory needs a reachability note or an explicit statement that reachability was not checked." ], "out_of_scope": [ "Exploiting a weakness against any running system.", "Disclosing a weakness to anyone other than the requester." ], "privacy": "Findings go only to the requester. Never contact the project's maintainers, users or services." }, "relations": [ [ "requires_noun", "evidence-standard" ] ] }, { "slug": "family.speed", "category": "noun.skill_pack", "role": "family_guide", "title": "Investigation guide: speed", "category_reason": "A family guide is a reusable investigation method with its guardrails, used by every worker that investigates this family.", "value": { "family": "speed", "scope": "How fast people and the system move: time to first success, CI and test duration, request latency, cold starts, page weight, release and review latency, and the local development loop.", "look_at": [ "README setup steps and dev scripts.", "CI history and step timings.", "Test runner output with durations.", "Request handlers and data access code.", "Release workflows and pull request timestamps." ], "cautions": [ "Every timing needs its method, environment, hardware and sample count.", "Measure in an isolated sandbox or use existing logs; never load-test production.", "One run is a sample, not a distribution; report the spread when you have two or more runs." ], "out_of_scope": [ "Load or stress testing any shared or production service.", "Changing project settings to improve a measurement." ], "privacy": "Findings go only to the requester. Never contact the project's maintainers, users or services." }, "relations": [ [ "requires_noun", "evidence-standard" ] ] }, { "slug": "family.reach-distribution", "category": "noun.skill_pack", "role": "family_guide", "title": "Investigation guide: reach and distribution", "category_reason": "A family guide is a reusable investigation method with its guardrails, used by every worker that investigates this family.", "value": { "family": "reach-distribution", "scope": "Whether the people who need the project can find it, understand it, install it and succeed with it: README first screen, install channels, metadata, docs, integrations, license, accessibility, demand signals and demos.", "look_at": [ "README, docs and examples.", "Package metadata and published channels.", "Repository metadata and public issues.", "Demo assets and the evidence behind their numbers." ], "cautions": [ "Public metrics are signals with a retrieval date, not proof of value.", "Judge clarity against the project's own stated audience.", "Do not post, star, comment, publish or contact anyone." ], "out_of_scope": [ "Promoting the project.", "Editing metadata or opening issues." ], "privacy": "Findings go only to the requester. Never contact the project's maintainers, users or services." }, "relations": [ [ "requires_noun", "evidence-standard" ] ] }, { "slug": "family.revenue", "category": "noun.skill_pack", "role": "family_guide", "title": "Investigation guide: revenue", "category_reason": "A family guide is a reusable investigation method with its guardrails, used by every worker that investigates this family.", "value": { "family": "revenue", "scope": "Whether the project could sustain itself: who pays, the pricing unit, cost to serve, the free and paid boundary, payment safety, funding programs, adoption requirements and heavy-user risk.", "look_at": [ "The stated audience and user evidence.", "Cost drivers in code.", "Payment integration code.", "Comparable products' public pricing, recorded with dates." ], "cautions": [ "Label speculation as speculation; market sizes and willingness to pay need sources.", "Use dated list prices and the requester's supplied numbers; never invent revenue.", "Do not apply to programs, create accounts, make payments or contact anyone." ], "out_of_scope": [ "Legal, tax or investment advice.", "Any real payment or live-mode payment test." ], "privacy": "Findings go only to the requester. Never contact the project's maintainers, users or services." }, "relations": [ [ "requires_noun", "evidence-standard" ] ] }, { "slug": "evidence-standard", "category": "noun.benchmark_suite", "role": "evidence_standard", "title": "Evidence standard and deterministic checks for repository findings", "category_reason": "It holds a scoring rubric, deterministic checks and golden fixtures with recorded expected results.", "value": { "fixture_kind": "finding_checks", "purpose": "What counts as evidence for a finding, and checks every finding must pass before a reviewer reads it.", "evidence_kinds": { "file": "A path and line range at the pinned commit, with a short excerpt.", "command": "A command, its working directory and the relevant output lines.", "source": "A public URL with the date it was read.", "measurement": "A number from a stated method in a stated environment with a sample count." }, "number_rules": [ "Every number has a unit, a method and a basis: measured, estimated or reported.", "Estimates show their arithmetic and inputs.", "Reported numbers name who reported them." ], "confidence_levels": { "high": "Direct evidence at the pinned commit, or a measurement repeated at least twice.", "medium": "Direct evidence with one stated assumption.", "low": "Indirect evidence; say what would raise confidence." }, "verdicts": { "concern_confirmed": "The evidence shows the concern applies.", "no_concern": "The evidence shows the concern does not apply. This is a useful result.", "not_applicable": "The question does not fit this repository; say why.", "unknown": "The permitted evidence cannot settle it; say what would." }, "privacy_rules": [ "Findings go only to the requester.", "Never contact the reviewed project's maintainers, users or services.", "Mask secrets and never include personal data." ], "finding_schema": { "type": "object", "additionalProperties": false, "required": [ "question_id", "verdict", "statement", "evidence", "numbers", "confidence" ], "properties": { "question_id": { "type": "string", "pattern": "^[a-z-]+\\.[a-z0-9-]+$" }, "verdict": { "type": "string", "enum": [ "concern_confirmed", "no_concern", "not_applicable", "unknown" ] }, "statement": { "type": "string", "minLength": 1, "maxLength": 800 }, "evidence": { "type": "array", "maxItems": 12, "items": { "type": "object", "additionalProperties": false, "required": [ "kind", "location" ], "properties": { "kind": { "type": "string", "enum": [ "file", "command", "source", "measurement" ] }, "location": { "type": "string", "minLength": 1, "maxLength": 300 }, "excerpt": { "type": "string", "maxLength": 600 }, "observed_at": { "type": "string", "maxLength": 40 } } } }, "numbers": { "type": "array", "maxItems": 12, "items": { "type": "object", "additionalProperties": false, "required": [ "value", "unit", "method", "basis" ], "properties": { "value": { "type": "number" }, "unit": { "type": "string", "minLength": 1, "maxLength": 60 }, "method": { "type": "string", "minLength": 1, "maxLength": 400 }, "basis": { "type": "string", "enum": [ "measured", "estimated", "reported" ] } } } }, "impact": { "type": "string", "enum": [ "high", "medium", "low", "unknown" ] }, "effort": { "type": "string", "enum": [ "small", "medium", "large", "unknown" ] }, "confidence": { "type": "string", "enum": [ "high", "medium", "low" ] }, "next_step": { "type": "string", "maxLength": 600 }, "unknown_reason": { "type": "string", "maxLength": 600 } } }, "deterministic_checks": [ { "id": "verdict-valid", "rule": { "rule": "enum", "path": "verdict", "values": [ "concern_confirmed", "no_concern", "not_applicable", "unknown" ] }, "why": "Only the four defined verdicts are comparable across reviews." }, { "id": "statement-present", "rule": { "rule": "present", "path": "statement" }, "why": "A finding says what was found in one plain sentence." }, { "id": "evidence-for-verdict", "rule": { "rule": "when", "if": { "path": "verdict", "in": [ "concern_confirmed", "no_concern" ] }, "then": { "rule": "each_has", "path": "evidence", "fields": [ "kind", "location" ], "min": 1 } }, "why": "Confirming or ruling out a concern needs at least one located piece of evidence." }, { "id": "numbers-qualified", "rule": { "rule": "each_has", "path": "numbers", "fields": [ "unit", "method", "basis" ] }, "why": "Every number has a unit, a method and a basis." }, { "id": "unknown-explained", "rule": { "rule": "when", "if": { "path": "verdict", "equals": "unknown" }, "then": { "rule": "present", "path": "unknown_reason" } }, "why": "An unknown verdict says what is missing and what would resolve it." }, { "id": "next-step-for-concern", "rule": { "rule": "when", "if": { "path": "verdict", "equals": "concern_confirmed" }, "then": { "rule": "present", "path": "next_step" } }, "why": "A confirmed concern names one concrete next step." }, { "id": "no-outreach", "rule": { "rule": "pattern_absent", "path": "$", "pattern": "(?i)\\b(contact|e-?mail|notify|ping|reach out to|open an? issue|file an? issue|open a pull request|submit a pull request|send a message to)\\b[^.]{0,40}\\b(maintainers?|authors?|owners?|developers?|team)\\b" }, "why": "Findings stay inside the requester's scope; no step contacts the reviewed project's people." }, { "id": "no-secret-values", "rule": { "rule": "pattern_absent", "path": "$", "pattern": "(?i:\\b(?:api[_-]?key|secret|token|password|passwd|pwd|access[_-]?key|client[_-]?secret)\\b[\"']?\\s*[:=]\\s*[\"']?[A-Za-z0-9_\\-/+=.]{8,})|(?i:\\bbearer\\s+[A-Za-z0-9_\\-.=+/]{12,})|\\b[A-Z0-9_]*(?:SECRET|TOKEN|PASSWORD|API_KEY)[A-Z0-9_]*\\s*=\\s*[^\\s\"']{8,}" }, "why": "Report where a secret is, never its value." } ], "golden_fixtures": [ { "id": "good-ci-minutes", "finding": { "question_id": "cost.ci-minutes", "verdict": "concern_confirmed", "statement": "The full test matrix runs on documentation-only pushes; 9 of the last 24 runs changed only docs.", "evidence": [ { "kind": "file", "location": ".github/workflows/test.yml:3-9", "excerpt": "on: [push, pull_request] with no paths filter" }, { "kind": "command", "location": "read-only run history export, runs 1001-1024", "excerpt": "9 runs touched only docs/ and *.md" } ], "numbers": [ { "value": 126, "unit": "runner-minutes per 24 runs", "method": "Sum of job durations for the 9 docs-only runs in the export", "basis": "measured" } ], "impact": "medium", "effort": "small", "confidence": "high", "next_step": "Add a paths-ignore filter for docs/** and *.md to the test workflow and keep a lightweight docs check." }, "label": "known_good" }, { "id": "good-unknown-latency", "finding": { "question_id": "speed.hot-path-latency", "verdict": "unknown", "statement": "Latency of the search endpoint could not be measured.", "evidence": [ { "kind": "file", "location": "src/search.py:40-88", "excerpt": "handler performs two queries per request" } ], "numbers": [], "confidence": "low", "unknown_reason": "Sandboxed runs were not permitted and no request logs were supplied; a local run with 200 samples would resolve it." }, "label": "known_good" }, { "id": "bad-no-evidence", "finding": { "question_id": "cost.idle-compute", "verdict": "concern_confirmed", "statement": "The worker probably costs about 40 dollars a month while idle.", "evidence": [], "numbers": [ { "value": 40, "unit": "USD per month", "method": "Typical for this kind of service", "basis": "estimated" } ], "confidence": "medium", "next_step": "Scale the worker to zero." }, "label": "known_bad" }, { "id": "bad-unqualified-number", "finding": { "question_id": "speed.ci-duration", "verdict": "concern_confirmed", "statement": "CI is slow.", "evidence": [ { "kind": "command", "location": "run history export" } ], "numbers": [ { "value": 18, "unit": "", "method": "", "basis": "measured" } ], "confidence": "medium", "next_step": "Split the integration job." }, "label": "known_bad" }, { "id": "bad-outreach", "finding": { "question_id": "reliability-security.ci-supply-chain", "verdict": "concern_confirmed", "statement": "A workflow runs fork code with repository secrets.", "evidence": [ { "kind": "file", "location": ".github/workflows/label.yml:2", "excerpt": "on: pull_request_target" } ], "numbers": [], "confidence": "high", "next_step": "Open an issue with the maintainers describing the problem." }, "label": "known_bad" }, { "id": "bad-secret-value", "finding": { "question_id": "reliability-security.secret-exposure", "verdict": "concern_confirmed", "statement": "A deploy script echoes its credential.", "evidence": [ { "kind": "file", "location": "deploy.sh:14", "excerpt": "echo token = abcd1234efgh5678wxyz" } ], "numbers": [], "confidence": "high", "next_step": "Remove the echo and rotate the credential." }, "label": "known_bad" }, { "id": "bad-quoted-secret", "finding": { "question_id": "reliability-security.secret-exposure", "verdict": "concern_confirmed", "statement": "A deploy script echoes its credential.", "evidence": [ { "kind": "file", "location": "config/app.json:7", "excerpt": "sets \"token\": \"abcd1234efgh5678\" for the API" } ], "numbers": [], "confidence": "high", "next_step": "Remove the echo and rotate the credential." }, "label": "known_bad" }, { "id": "bad-bearer-header", "finding": { "question_id": "reliability-security.secret-exposure", "verdict": "concern_confirmed", "statement": "A deploy script echoes its credential.", "evidence": [ { "kind": "file", "location": "app/log.py:22", "excerpt": "logs Authorization: Bearer abcdefghijklmnop1234" } ], "numbers": [], "confidence": "high", "next_step": "Remove the echo and rotate the credential." }, "label": "known_bad" }, { "id": "bad-env-secret", "finding": { "question_id": "reliability-security.secret-exposure", "verdict": "concern_confirmed", "statement": "A deploy script echoes its credential.", "evidence": [ { "kind": "file", "location": "deploy.env:2", "excerpt": "AWS_SECRET_ACCESS_KEY=notarealvalue1234" } ], "numbers": [], "confidence": "high", "next_step": "Remove the echo and rotate the credential." }, "label": "known_bad" } ], "expected": { "good-ci-minutes": { "verdict": "pass", "failed_checks": [] }, "good-unknown-latency": { "verdict": "pass", "failed_checks": [] }, "bad-no-evidence": { "verdict": "fail", "failed_checks": [ "evidence-for-verdict" ] }, "bad-unqualified-number": { "verdict": "fail", "failed_checks": [ "numbers-qualified" ] }, "bad-outreach": { "verdict": "fail", "failed_checks": [ "no-outreach" ] }, "bad-secret-value": { "verdict": "fail", "failed_checks": [ "no-secret-values" ] }, "bad-quoted-secret": { "verdict": "fail", "failed_checks": [ "no-secret-values" ] }, "bad-bearer-header": { "verdict": "fail", "failed_checks": [ "no-secret-values" ] }, "bad-env-secret": { "verdict": "fail", "failed_checks": [ "no-secret-values" ] } } }, "relations": [ [ "requires_noun", "template.finding" ] ] }, { "slug": "question.cost.idle-compute", "category": "noun.seed_hypothesis", "role": "question", "title": "Which services, runners or scheduled jobs keep running or polling when nobody uses them, and what does each cost per month?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.cost" ] ], "value": { "id": "cost.idle-compute", "family": "cost", "question": "Which services, runners or scheduled jobs keep running or polling when nobody uses them, and what does each cost per month?", "why_it_matters": "Always-on capacity and frequent polling often dominate a small project's bill without anyone noticing.", "applies_when": [ "Deployment config exists: containers, serverless functions, virtual machines, workers or schedulers.", "Cron schedules, polling loops or minimum-instance settings appear in code or config." ], "skip_when": [ "The repository is a library or tool with no deployment of its own." ], "evidence_required": [ "Each always-on or scheduled component with file path and line.", "Its trigger, cadence or minimum-instance setting.", "The provider's list price for that unit, with the date it was read.", "Usage numbers only if the requester supplied them; otherwise mark usage unknown." ], "falsified_by": "Every component scales to zero or runs only on demand, or its monthly cost is below the requester's stated threshold.", "rubric": { "strong_answer": [ "Lists every scheduled or always-on component with its trigger and cadence.", "Shows the monthly arithmetic with units and a dated price source.", "Keeps measured usage, list-price estimates and assumptions visibly separate." ], "fails_if": [ "Quotes a monthly cost without units or a price source.", "Assumes traffic the requester did not supply." ] }, "unknown_when": "Pricing depends on an account plan or usage figures that the requester has not supplied.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" }, { "$verb": "cost-optimization.propose-savings" } ], "measure": "Monthly cost per component equals units per month times the dated list price." } }, { "slug": "question.cost.ci-minutes", "category": "noun.seed_hypothesis", "role": "question", "title": "Which CI jobs run on changes that cannot affect their result, and how many runner-minutes would path filters, concurrency cancellation or caching save?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.cost" ] ], "value": { "id": "cost.ci-minutes", "family": "cost", "question": "Which CI jobs run on changes that cannot affect their result, and how many runner-minutes would path filters, concurrency cancellation or caching save?", "why_it_matters": "CI minutes are a recurring cost and a queue that slows everyone; docs-only changes rarely need the full matrix.", "applies_when": [ "CI workflow files exist.", "Recent run history is readable through the CI provider's read-only interface or an export." ], "skip_when": [], "evidence_required": [ "Workflow triggers, path filters and concurrency settings with file path and line.", "Durations of at least 20 recent runs per job, with their run identifiers.", "The changed paths of those runs, showing which could not affect the job." ], "falsified_by": "Path filters and cancellation already exclude unaffected runs, or total CI time is negligible on the requester's plan.", "rubric": { "strong_answer": [ "Classifies at least 20 recent runs by the paths they changed.", "Computes avoidable runner-minutes with the formula shown.", "Names the exact filter, concurrency group or cache to add." ], "fails_if": [ "Recommends skipping tests without showing that the changed paths cannot affect them." ] }, "unknown_when": "Run history is not readable with the permitted access.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" } ], "measure": "Runner-minutes per month spent on runs whose changed paths cannot affect the job." } }, { "slug": "question.cost.dependency-caching", "category": "noun.seed_hypothesis", "role": "question", "title": "Does CI reinstall dependencies, rebuild containers or download models on every run when a cache keyed on the lockfile would do?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.cost" ] ], "value": { "id": "cost.dependency-caching", "family": "cost", "question": "Does CI reinstall dependencies, rebuild containers or download models on every run when a cache keyed on the lockfile would do?", "why_it_matters": "Repeated installs waste minutes and bandwidth and make CI slower and flakier.", "applies_when": [ "CI installs packages, builds images or downloads large files." ], "skip_when": [], "evidence_required": [ "The install, build or download steps with file path and line.", "Cache configuration and the exact cache key expression.", "Step durations and cache hit or miss lines from recent logs." ], "falsified_by": "A cache keyed on the lockfile or content hash already hits on most runs.", "rubric": { "strong_answer": [ "Quotes the cache key and shows that it changes only when the inputs change.", "Reports the observed hit rate from recent logs." ], "fails_if": [ "Suggests caching files that contain secrets or per-run state." ] }, "unknown_when": "Logs that show step timing or cache behavior are unavailable.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" } ], "measure": "Median seconds per run spent installing or building, and the share a cache hit would remove." } }, { "slug": "question.cost.model-spend", "category": "noun.seed_hypothesis", "role": "question", "title": "Where does the code call paid model or provider APIs, with which model, output limit, retry policy and per-run allowance, and is actual usage recorded?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.cost" ] ], "value": { "id": "cost.model-spend", "family": "cost", "question": "Where does the code call paid model or provider APIs, with which model, output limit, retry policy and per-run allowance, and is actual usage recorded?", "why_it_matters": "Model calls can be the largest variable cost, and unrecorded usage hides both waste and failures.", "applies_when": [ "The code calls a hosted model, speech, image or other metered API." ], "skip_when": [], "evidence_required": [ "Every call site with file path and line, including fallbacks.", "Configured model identifiers, output limits and retry or failover settings.", "Where usage, cost or failed attempts are recorded, or that they are not.", "Dated list prices for the configured models." ], "falsified_by": "There are no paid calls, or every call path has an allowance and records usage and failures.", "rubric": { "strong_answer": [ "Inventories every call path, including failover and retries.", "Computes a worst case from configured limits, not from typical use.", "States whether failed or uncertain attempts are recorded." ], "fails_if": [ "Reads, prints or tests credential values.", "Recommends lowering the caller's chosen output settings without evidence that they waste money." ] }, "unknown_when": "Usage records or prices needed for the estimate are not available to the reviewer.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" }, { "$verb": "cost-optimization.propose-savings" } ], "measure": "Worst-case paid calls per user action times the dated price per call at the configured limits." } }, { "slug": "question.cost.retry-amplification", "category": "noun.seed_hypothesis", "role": "question", "title": "Can one user action trigger repeated paid work through retries, failover chains or loops with no overall ceiling?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.cost" ] ], "value": { "id": "cost.retry-amplification", "family": "cost", "question": "Can one user action trigger repeated paid work through retries, failover chains or loops with no overall ceiling?", "why_it_matters": "Unbounded retries turn one outage into a large bill and can repeat external effects.", "applies_when": [ "The code retries network calls, fails over between providers or loops over remote work." ], "skip_when": [], "evidence_required": [ "The retry, backoff and failover code with file path and line.", "The loop or recursion bounds that apply.", "One traced worst-case path from a user action to its maximum number of paid attempts." ], "falsified_by": "Every path has an explicit overall ceiling that is enforced in code and covered by a test.", "rubric": { "strong_answer": [ "Traces one worst-case path end to end.", "Points to the exact bound or shows that it is missing." ], "fails_if": [ "Claims a bound exists without pointing to the code that enforces it." ] }, "unknown_when": "The call graph depends on runtime configuration that the snapshot does not include.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" } ], "measure": "Maximum paid attempts reachable from one user action." } }, { "slug": "question.cost.storage-growth", "category": "noun.seed_hypothesis", "role": "question", "title": "Which stores grow without retention, such as logs, audit rows, artifacts, caches or uploads, and how fast do they grow per day or per active user?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.cost" ] ], "value": { "id": "cost.storage-growth", "family": "cost", "question": "Which stores grow without retention, such as logs, audit rows, artifacts, caches or uploads, and how fast do they grow per day or per active user?", "why_it_matters": "Unbounded growth raises storage bills and slows queries long before anyone deletes data.", "applies_when": [ "The project writes to a database, object store, log sink or cache." ], "skip_when": [], "evidence_required": [ "Each write path with file path and line.", "Retention, TTL or cleanup settings, or their absence.", "Rows or bytes added per operation, measured in a local run when possible." ], "falsified_by": "Every growing store has a retention rule, a size cap or a documented reason to keep data.", "rubric": { "strong_answer": [ "Lists each growing store with its retention rule or its absence.", "Measures growth per operation instead of guessing." ], "fails_if": [ "Recommends deleting data without noting legal, audit or user expectations." ] }, "unknown_when": "Growth depends on production traffic that is not available.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" }, { "$verb": "cost-optimization.propose-savings" } ], "measure": "Rows or bytes added per operation in a local run, times operations per day if the requester supplied them." } }, { "slug": "question.cost.per-request-operations", "category": "noun.seed_hypothesis", "role": "question", "title": "How many billable operations, such as database reads and writes, key-value operations, egress or function invocations, does one typical request cause, and which could be cached, batched or skipped?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.cost" ] ], "value": { "id": "cost.per-request-operations", "family": "cost", "question": "How many billable operations, such as database reads and writes, key-value operations, egress or function invocations, does one typical request cause, and which could be cached, batched or skipped?", "why_it_matters": "Per-request multipliers decide whether the project fits a free tier or a budget.", "applies_when": [ "The project serves requests through a metered platform or database." ], "skip_when": [], "evidence_required": [ "The handler path for the top requests with file path and line.", "Operations counted in a local test with logging or instrumentation.", "The provider's billing unit for each operation, with the date it was read." ], "falsified_by": "The top requests already use the minimum operations their behavior needs.", "rubric": { "strong_answer": [ "Counts operations for the top requests in a local run.", "Names the write or read that could be skipped, cached or coalesced." ], "fails_if": [ "Counts operations without stating the provider's billing unit." ] }, "unknown_when": "The code cannot be run locally and the request path cannot be traced statically.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" }, { "$verb": "cost-optimization.propose-savings" } ], "measure": "Billable operations per request, counted in a local run." } }, { "slug": "question.cost.plan-limit-cliffs", "category": "noun.seed_hypothesis", "role": "question", "title": "Which free-tier or plan limits is the project close to, and what happens when one is hit: graceful degradation, hard failure or surprise billing?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.cost" ] ], "value": { "id": "cost.plan-limit-cliffs", "family": "cost", "question": "Which free-tier or plan limits is the project close to, and what happens when one is hit: graceful degradation, hard failure or surprise billing?", "why_it_matters": "Limits fail suddenly; knowing the cliff and the failure mode in advance prevents an outage.", "applies_when": [ "The project depends on a hosted platform with published plan limits." ], "skip_when": [], "evidence_required": [ "The specific limit, its value and the source with the date it was read.", "Current usage only if the requester supplied it.", "The code path that runs when the limit error occurs, and any alert." ], "falsified_by": "Usage is far below the limits, and limit errors are handled and alerted.", "rubric": { "strong_answer": [ "Names the limit, its value and the dated source.", "Shows the code path that handles the limit error." ], "fails_if": [ "Recommends an account or credential workaround to evade a provider limit." ] }, "unknown_when": "Usage figures are unavailable, so distance from the limit is unknown.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" }, { "$verb": "cost-optimization.propose-savings" } ], "measure": "Headroom per limit as a share of the limit, when usage is supplied." } }, { "slug": "question.cost.unused-services", "category": "noun.seed_hypothesis", "role": "question", "title": "Which paid services, keys, domains or software subscriptions are configured but no longer used by the code?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.cost" ] ], "value": { "id": "cost.unused-services", "family": "cost", "question": "Which paid services, keys, domains or software subscriptions are configured but no longer used by the code?", "why_it_matters": "Forgotten services keep charging and keep credentials alive that could leak.", "applies_when": [ "The project references environment variables, service configs or third-party integrations." ], "skip_when": [], "evidence_required": [ "Each configured name (never its value) and where it is defined.", "Whether code references it, with file path and line, or that no reference was found.", "Docs or deploy manifests that mention it." ], "falsified_by": "Every configured service and key is referenced by live code or documented as intentionally reserved.", "rubric": { "strong_answer": [ "Cross-references each configured name with code usage.", "Marks uncertain cases unknown instead of unused." ], "fails_if": [ "Opens or prints secret values.", "Recommends deleting a credential instead of reporting it to the requester." ] }, "unknown_when": "Configuration lives outside the snapshot, for example in a hosting dashboard.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" } ], "measure": "Count of configured services with no code reference." } }, { "slug": "question.cost.artifact-size", "category": "noun.seed_hypothesis", "role": "question", "title": "How large are the build artifacts, container images or bundles, and does their size drive hosting, transfer or cold-start cost?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.cost" ] ], "value": { "id": "cost.artifact-size", "family": "cost", "question": "How large are the build artifacts, container images or bundles, and does their size drive hosting, transfer or cold-start cost?", "why_it_matters": "Large artifacts cost money to store and move and slow every deploy and cold start.", "applies_when": [ "The project builds a container, package, bundle or binary." ], "skip_when": [], "evidence_required": [ "Sizes measured from a local build in a sandbox, with the build command.", "The three largest contributors, from a layer or bundle breakdown.", "Any platform size limit that applies, with its source and date." ], "falsified_by": "Artifacts are small relative to platform limits and contain no avoidable weight.", "rubric": { "strong_answer": [ "Measures the artifact instead of estimating it.", "Names the top contributors and a concrete reduction." ], "fails_if": [ "Recommends removing files without checking that they are unused at runtime." ] }, "unknown_when": "A local build is not permitted or fails, so sizes cannot be measured.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" } ], "measure": "Bytes per artifact and its top three contributors." } }, { "slug": "question.reliability-security.secret-exposure", "category": "noun.seed_hypothesis", "role": "question", "title": "Could a credential be committed, logged, echoed in an error or returned to a client?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.reliability-security" ] ], "value": { "id": "reliability-security.secret-exposure", "family": "reliability-security", "question": "Could a credential be committed, logged, echoed in an error or returned to a client?", "why_it_matters": "One leaked key can cost more than the whole project and is hard to take back.", "applies_when": [ "The project handles API keys, tokens, passwords or signing keys." ], "skip_when": [], "evidence_required": [ "Secret-scanning results with locations and masked values.", "Logging of headers, environment variables or configuration, with file path and line.", "Error handlers or debug endpoints that serialize configuration.", "Ignore rules and CI log masking." ], "falsified_by": "No secret appears in tracked files or history in the snapshot, and logs and errors never include credentials.", "rubric": { "strong_answer": [ "Reports each location with the value masked.", "Checks history as well as the current tree when the snapshot includes history." ], "fails_if": [ "Prints any secret value beyond a short masked prefix.", "Tests a found credential against a live service." ] }, "unknown_when": "The snapshot excludes history or generated logs needed to answer.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "security-review.review-code" } ] } }, { "slug": "question.reliability-security.authorization", "category": "noun.seed_hypothesis", "role": "question", "title": "For every route that writes data or reads private data, is authorization checked on the server against the exact resource, and is there a test showing that another user is refused?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.reliability-security" ] ], "value": { "id": "reliability-security.authorization", "family": "reliability-security", "question": "For every route that writes data or reads private data, is authorization checked on the server against the exact resource, and is there a test showing that another user is refused?", "why_it_matters": "Missing object-level checks are among the most common and damaging web flaws.", "applies_when": [ "The project exposes an HTTP, RPC, MCP or other remote interface with more than one user or role." ], "skip_when": [ "The project has no remote interface or only one user." ], "evidence_required": [ "A table of sensitive routes, each with its authorization check (file path and line).", "Tests that a different user or tenant is refused, or their absence.", "Routes that rely on client-side checks, hidden URLs or guessable identifiers." ], "falsified_by": "Every sensitive route checks the caller's right to the exact resource on the server, with a denial test.", "rubric": { "strong_answer": [ "Tabulates every sensitive route with its check and its test.", "Identifies routes that depend on client-side checks or unguessable links." ], "fails_if": [ "Probes a live deployment instead of reading code and running local tests." ] }, "unknown_when": "Route registration is dynamic and cannot be enumerated from the snapshot.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" }, { "$verb": "security-review.review-code" } ] } }, { "slug": "question.reliability-security.input-validation", "category": "noun.seed_hypothesis", "role": "question", "title": "Are external inputs, such as request bodies, uploads, webhooks, files and model outputs, validated for type and size before use?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.reliability-security" ] ], "value": { "id": "reliability-security.input-validation", "family": "reliability-security", "question": "Are external inputs, such as request bodies, uploads, webhooks, files and model outputs, validated for type and size before use?", "why_it_matters": "Unvalidated input causes crashes, injection and runaway costs.", "applies_when": [ "The project accepts data from users, other services, files or models." ], "skip_when": [], "evidence_required": [ "Each input boundary with its validation code, file path and line.", "Size and rate limits, or their absence.", "Tests with malformed, oversized or wrong-type input." ], "falsified_by": "Every boundary validates type and size before use and has a malformed-input test.", "rubric": { "strong_answer": [ "Lists each boundary with its validator and limits.", "Shows a malformed-input test or a sandboxed run that exercises one." ], "fails_if": [ "Treats model output as trusted input without saying so." ] }, "unknown_when": "Inputs arrive through infrastructure configuration outside the snapshot.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" }, { "$verb": "security-review.review-code" } ] } }, { "slug": "question.reliability-security.idempotency", "category": "noun.seed_hypothesis", "role": "question", "title": "Can a retried request, a redelivered webhook or a replayed job repeat an external effect such as a charge, an email or a publication?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.reliability-security" ] ], "value": { "id": "reliability-security.idempotency", "family": "reliability-security", "question": "Can a retried request, a redelivered webhook or a replayed job repeat an external effect such as a charge, an email or a publication?", "why_it_matters": "Networks retry. Without idempotency, users get charged or emailed twice.", "applies_when": [ "The project performs external effects: payments, messages, publications or writes to other systems." ], "skip_when": [], "evidence_required": [ "Each external effect with file path and line.", "Its idempotency key, deduplication record or replay check, or their absence.", "Tests that a repeated delivery does not repeat the effect." ], "falsified_by": "Every external effect is keyed and deduplicated, with a replay test.", "rubric": { "strong_answer": [ "Traces each effect from trigger to provider call.", "Shows the key or dedupe store and its test." ], "fails_if": [ "Recommends retries that would repeat an effect whose outcome is uncertain." ] }, "unknown_when": "Effects happen in external services whose behavior the snapshot does not show.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" } ] } }, { "slug": "question.reliability-security.dependency-health", "category": "noun.seed_hypothesis", "role": "question", "title": "Which dependencies are unpinned, abandoned or affected by known advisories, and does CI install from a lockfile?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.reliability-security" ] ], "value": { "id": "reliability-security.dependency-health", "family": "reliability-security", "question": "Which dependencies are unpinned, abandoned or affected by known advisories, and does CI install from a lockfile?", "why_it_matters": "Supply-chain problems arrive through dependencies; lockfiles make builds reproducible.", "applies_when": [ "The project declares third-party dependencies." ], "skip_when": [], "evidence_required": [ "Lockfiles and whether CI installs with frozen or locked flags (file path and line).", "Advisory results with the tool name, its version and the date of the database.", "Dependencies with no release for a long time, with their last release date." ], "falsified_by": "All dependencies are locked, CI enforces the lockfile, and no advisory affects a reachable path.", "rubric": { "strong_answer": [ "Names the tool, version and database date for each advisory.", "Says for each advisory whether the vulnerable path is reachable, or that reachability was not checked." ], "fails_if": [ "Reports an advisory as exploitable without checking reachability or saying it was not checked." ] }, "unknown_when": "No offline advisory database is available to the reviewer.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "security-review.review-code" } ] } }, { "slug": "question.reliability-security.dependency-failure", "category": "noun.seed_hypothesis", "role": "question", "title": "When each external dependency, such as a database, model provider or third-party API, is slow, down or rate-limited, is there a timeout, a bounded retry that honors Retry-After, and a visible degraded state?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.reliability-security" ] ], "value": { "id": "reliability-security.dependency-failure", "family": "reliability-security", "question": "When each external dependency, such as a database, model provider or third-party API, is slow, down or rate-limited, is there a timeout, a bounded retry that honors Retry-After, and a visible degraded state?", "why_it_matters": "Outages of other services become your outage unless the code expects them.", "applies_when": [ "The project calls external services or databases." ], "skip_when": [], "evidence_required": [ "Timeouts and retry policies per dependency, with file path and line.", "Handling of rate-limit responses and Retry-After.", "What the user sees when the dependency fails.", "Tests or a sandboxed fault injection, if permitted." ], "falsified_by": "Every dependency call has a timeout, a bounded retry and a tested degraded state.", "rubric": { "strong_answer": [ "Tabulates each dependency with timeout, retry bound and failure behavior.", "Demonstrates one failure in a sandbox or a test." ], "fails_if": [ "Tests failure handling against a live third-party service." ] }, "unknown_when": "Client libraries hide their defaults and the versions in use are unknown.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" } ] } }, { "slug": "question.reliability-security.critical-path-tests", "category": "noun.seed_hypothesis", "role": "question", "title": "Which critical paths, such as authentication, payments, deletion, migrations and permission changes, have no test that would fail if they broke?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.reliability-security" ] ], "value": { "id": "reliability-security.critical-path-tests", "family": "reliability-security", "question": "Which critical paths, such as authentication, payments, deletion, migrations and permission changes, have no test that would fail if they broke?", "why_it_matters": "Untested critical paths break silently during refactors.", "applies_when": [ "The project has authentication, money, destructive operations or schema migrations." ], "skip_when": [], "evidence_required": [ "A map from each critical path to the tests that cover it, with file paths.", "A coverage report from a local sandboxed run, if one can be produced.", "Optionally, a deliberately broken line in a sandbox copy that no test catches." ], "falsified_by": "Each critical path has a test that fails when the path breaks.", "rubric": { "strong_answer": [ "Maps every critical path to specific tests or states that none exists.", "Shows at least one sandboxed break that tests catch or miss." ], "fails_if": [ "Equates line coverage with correctness." ] }, "unknown_when": "Tests cannot run in a sandbox, so coverage is unknown.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" } ] } }, { "slug": "question.reliability-security.backup-restore", "category": "noun.seed_hypothesis", "role": "question", "title": "Is there a tested way to restore stored data, and when was a restore last exercised?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.reliability-security" ] ], "value": { "id": "reliability-security.backup-restore", "family": "reliability-security", "question": "Is there a tested way to restore stored data, and when was a restore last exercised?", "why_it_matters": "Backups that were never restored are hopes, not backups.", "applies_when": [ "The project stores data that users or the project would miss." ], "skip_when": [ "The project stores no data of its own." ], "evidence_required": [ "Backup configuration or scripts, with file path and line.", "Restore documentation or scripts.", "Evidence of a past restore test, with its date, or its absence." ], "falsified_by": "Backups exist and a documented restore was exercised recently.", "rubric": { "strong_answer": [ "Distinguishes backup existence from a tested restore.", "Gives the date of the last restore test or says none is recorded." ], "fails_if": [ "Runs a restore against production data." ] }, "unknown_when": "Backups are managed by a hosting provider outside the snapshot.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" } ] } }, { "slug": "question.reliability-security.rollback", "category": "noun.seed_hypothesis", "role": "question", "title": "Can a bad deploy be rolled back quickly to a recorded previous version, and are database migrations reversible or forward-compatible?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.reliability-security" ] ], "value": { "id": "reliability-security.rollback", "family": "reliability-security", "question": "Can a bad deploy be rolled back quickly to a recorded previous version, and are database migrations reversible or forward-compatible?", "why_it_matters": "Fast rollback turns a bad release into a short incident instead of a long one.", "applies_when": [ "The project deploys a service or database schema." ], "skip_when": [ "The project is not deployed." ], "evidence_required": [ "Deploy scripts and how the previous version is recorded.", "Rollback documentation or commands.", "Migration files and whether each has a down path or is backward compatible." ], "falsified_by": "Rollback is documented, the previous version is recorded, and migrations are reversible or compatible.", "rubric": { "strong_answer": [ "Names the exact rollback command and where the previous version is recorded.", "Classifies each recent migration as reversible, compatible or one-way." ], "fails_if": [ "Recommends deploying or rolling back anything as part of the review." ] }, "unknown_when": "Deploys happen through a dashboard outside the snapshot.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" } ] } }, { "slug": "question.reliability-security.observability", "category": "noun.seed_hypothesis", "role": "question", "title": "Would the maintainers learn about an outage or error spike before users report it?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.reliability-security" ] ], "value": { "id": "reliability-security.observability", "family": "reliability-security", "question": "Would the maintainers learn about an outage or error spike before users report it?", "why_it_matters": "Silent failures cost users and trust; detection time bounds recovery time.", "applies_when": [ "The project runs as a service, job or scheduled task." ], "skip_when": [ "The project is a library with no running service." ], "evidence_required": [ "Health checks, alert rules and error reporting configuration, with file path and line.", "Runbooks or on-call documentation.", "What signal would fire for the most likely failure." ], "falsified_by": "The likely failures each trigger an alert that reaches a person.", "rubric": { "strong_answer": [ "Names the signal and alert for the most likely failures.", "Identifies failures that would go unnoticed." ], "fails_if": [ "Triggers alerts or sends test notifications." ] }, "unknown_when": "Monitoring lives in a hosted dashboard outside the snapshot.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" } ] } }, { "slug": "question.reliability-security.ci-supply-chain", "category": "noun.seed_hypothesis", "role": "question", "title": "Do CI workflows run untrusted code with secrets or write permissions, use third-party actions without pinned commits, or grant broader token permissions than needed?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.reliability-security" ] ], "value": { "id": "reliability-security.ci-supply-chain", "family": "reliability-security", "question": "Do CI workflows run untrusted code with secrets or write permissions, use third-party actions without pinned commits, or grant broader token permissions than needed?", "why_it_matters": "CI is a privileged environment; a malicious pull request or action can steal secrets or push code.", "applies_when": [ "The project uses hosted CI with workflow files." ], "skip_when": [], "evidence_required": [ "Workflow triggers that run fork code with secrets, such as pull_request_target, with file path and line.", "Permission blocks and the default token scope.", "Third-party actions referenced by tag rather than by commit hash." ], "falsified_by": "Fork code never runs with secrets, permissions are minimal, and third-party actions are pinned.", "rubric": { "strong_answer": [ "Lists each workflow with trigger, permissions and action references.", "Explains the concrete risk for each flagged trigger." ], "fails_if": [ "Opens a pull request to test the weakness." ] }, "unknown_when": "Workflows are generated or stored outside the snapshot.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "security-review.review-code" } ] } }, { "slug": "question.reliability-security.personal-data", "category": "noun.seed_hypothesis", "role": "question", "title": "What personal or sensitive data does the project collect or store, and are minimization, retention and deletion implemented?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.reliability-security" ] ], "value": { "id": "reliability-security.personal-data", "family": "reliability-security", "question": "What personal or sensitive data does the project collect or store, and are minimization, retention and deletion implemented?", "why_it_matters": "Personal data creates legal duties and breach risk; collecting less is the strongest control.", "applies_when": [ "The project collects data about people: accounts, messages, analytics, uploads or logs with identifiers." ], "skip_when": [ "The project handles no data about people." ], "evidence_required": [ "The data models and fields that hold personal data, with file path and line.", "Retention and deletion code or settings.", "Privacy documentation and analytics configuration." ], "falsified_by": "Only needed data is collected, retention is limited, and deletion works and is tested.", "rubric": { "strong_answer": [ "Lists personal fields by store with their retention and deletion paths." ], "fails_if": [ "Copies real personal data into the finding." ] }, "unknown_when": "Data flows depend on third-party services whose configuration is not in the snapshot.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "security-review.review-code" } ] } }, { "slug": "question.speed.first-success", "category": "noun.seed_hypothesis", "role": "question", "title": "How long does a new contributor need from a fresh clone to a passing test run, following only the README?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.speed" ] ], "value": { "id": "speed.first-success", "family": "speed", "question": "How long does a new contributor need from a fresh clone to a passing test run, following only the README?", "why_it_matters": "Time to first success decides whether contributors and evaluators stay.", "applies_when": [ "The project documents setup and test commands.", "The requester allowed sandboxed runs." ], "skip_when": [], "evidence_required": [ "A timed run in a clean sandbox: each command, its duration and any failure.", "Operating system, toolchain versions and hardware class.", "Each place where the README was missing a step." ], "falsified_by": "A clean sandbox reaches a passing test run quickly by following the README exactly.", "rubric": { "strong_answer": [ "Times each step in a clean environment.", "Lists every missing or wrong README step." ], "fails_if": [ "Fixes the README silently during the run instead of reporting the gap." ] }, "unknown_when": "Sandboxed runs are not permitted or the environment cannot match the documented requirements.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" }, { "$verb": "evaluation-design.run-arm" } ], "measure": "Minutes from clone to the first passing test run, with the step that took longest." } }, { "slug": "question.speed.ci-duration", "category": "noun.seed_hypothesis", "role": "question", "title": "What are the median and slowest CI durations over the last 20 or more runs, and which job or step dominates?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.speed" ] ], "value": { "id": "speed.ci-duration", "family": "speed", "question": "What are the median and slowest CI durations over the last 20 or more runs, and which job or step dominates?", "why_it_matters": "Slow CI delays every change and multiplies cost.", "applies_when": [ "CI run history is readable." ], "skip_when": [], "evidence_required": [ "Durations for at least 20 recent runs, with run identifiers.", "Per-job and per-step timing for the slowest runs.", "The critical path through jobs that depend on each other." ], "falsified_by": "Median CI time is short for the project's size, with no single dominant step.", "rubric": { "strong_answer": [ "Reports median and slowest durations with the run count.", "Names the step on the critical path that dominates." ], "fails_if": [ "Reports one run as typical." ] }, "unknown_when": "Run history is not readable with the permitted access.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" } ], "measure": "Median and maximum CI minutes over at least 20 runs." } }, { "slug": "question.speed.test-suite", "category": "noun.seed_hypothesis", "role": "question", "title": "Which tests dominate suite time, and could they be parallelized, sharded or made hermetic and faster?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.speed" ] ], "value": { "id": "speed.test-suite", "family": "speed", "question": "Which tests dominate suite time, and could they be parallelized, sharded or made hermetic and faster?", "why_it_matters": "A slow suite gets skipped, and skipped tests stop protecting anyone.", "applies_when": [ "The project has an automated test suite.", "The requester allowed sandboxed runs, or CI logs include test timings." ], "skip_when": [], "evidence_required": [ "Per-test durations from a local run or CI logs, with the command.", "The slowest tests and why they are slow, such as sleeps, network or large fixtures.", "Whether the runner supports parallel execution." ], "falsified_by": "No small set of tests dominates and the suite already runs in parallel where safe.", "rubric": { "strong_answer": [ "Names the slowest tests with durations and causes.", "Proposes a specific change for each." ], "fails_if": [ "Recommends deleting slow tests without assessing what they protect." ] }, "unknown_when": "Tests cannot run and logs lack timings.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" }, { "$verb": "evaluation-design.run-arm" } ], "measure": "Share of suite time taken by the slowest 10 tests." } }, { "slug": "question.speed.hot-path-latency", "category": "noun.seed_hypothesis", "role": "question", "title": "What are the p50 and p95 latencies of the main user-facing operation, measured how, where and with how many samples?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.speed" ] ], "value": { "id": "speed.hot-path-latency", "family": "speed", "question": "What are the p50 and p95 latencies of the main user-facing operation, measured how, where and with how many samples?", "why_it_matters": "Latency people feel is the main speed fact for a service.", "applies_when": [ "The project serves interactive requests or runs a user-facing operation." ], "skip_when": [], "evidence_required": [ "The measurement method, environment, hardware and sample count.", "p50 and p95 from a sandboxed local run or from logs the requester supplied.", "The largest contributor found by profiling, if profiling was permitted." ], "falsified_by": "Measured latency meets a stated target, or there is no interactive path.", "rubric": { "strong_answer": [ "States method, environment and sample count with every number.", "Identifies the dominant contributor." ], "fails_if": [ "Reports latency without method, environment and sample count.", "Load-tests a production service." ] }, "unknown_when": "No sandboxed run is permitted and no logs were supplied.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" }, { "$verb": "evaluation-design.run-arm" } ], "measure": "p50 and p95 milliseconds over a stated number of samples." } }, { "slug": "question.speed.cold-start", "category": "noun.seed_hypothesis", "role": "question", "title": "How long do cold starts take, such as a serverless start, container boot or first model load, and how often would users meet one?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.speed" ] ], "value": { "id": "speed.cold-start", "family": "speed", "question": "How long do cold starts take, such as a serverless start, container boot or first model load, and how often would users meet one?", "why_it_matters": "Cold starts make the first experience the slowest one.", "applies_when": [ "The project runs on scale-to-zero infrastructure, boots containers on demand or loads large models." ], "skip_when": [], "evidence_required": [ "Measured cold start time in a sandbox, with method and samples.", "Configuration that controls warm capacity.", "What triggers a cold start and how often, if usage was supplied." ], "falsified_by": "Cold starts are short or rare for the expected traffic.", "rubric": { "strong_answer": [ "Measures cold and warm starts separately." ], "fails_if": [ "Keeps capacity warm as a fix without stating its cost." ] }, "unknown_when": "Cold starts cannot be reproduced outside the hosting platform.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" }, { "$verb": "evaluation-design.run-arm" } ], "measure": "Cold start seconds and warm start seconds, each with sample counts." } }, { "slug": "question.speed.n-plus-one", "category": "noun.seed_hypothesis", "role": "question", "title": "Do request handlers perform per-item queries or remote calls in loops that could be batched?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.speed" ] ], "value": { "id": "speed.n-plus-one", "family": "speed", "question": "Do request handlers perform per-item queries or remote calls in loops that could be batched?", "why_it_matters": "Per-item calls make latency and cost grow with data size.", "applies_when": [ "Handlers read collections from a database or remote API." ], "skip_when": [], "evidence_required": [ "Loops containing queries or remote calls, with file path and line.", "Query or call counts per request from a local run with logging." ], "falsified_by": "Every collection is fetched with a bounded number of queries regardless of size.", "rubric": { "strong_answer": [ "Shows the loop and the counted calls for two collection sizes." ], "fails_if": [ "Flags a loop without showing that it issues a call per item." ] }, "unknown_when": "Data access is generated at runtime and cannot be traced or run locally.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" } ] } }, { "slug": "question.speed.page-weight", "category": "noun.seed_hypothesis", "role": "question", "title": "How heavy is the first page load, counting scripts, styles, fonts and images, and what does a lab measurement report on a stated device profile?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.speed" ] ], "value": { "id": "speed.page-weight", "family": "speed", "question": "How heavy is the first page load, counting scripts, styles, fonts and images, and what does a lab measurement report on a stated device profile?", "why_it_matters": "Page weight decides whether people on slow connections and phones can use the surface.", "applies_when": [ "The project serves a web page or app." ], "skip_when": [], "evidence_required": [ "Transfer sizes by resource type from a local build or a lab tool.", "A lab measurement with the tool, version and device and network profile.", "The largest resources and whether they are needed for the first view." ], "falsified_by": "The first view is light for its purpose and the largest resources are needed.", "rubric": { "strong_answer": [ "Reports sizes by type with the tool and profile.", "Names the heaviest resources and a fix for each." ], "fails_if": [ "Runs the lab tool against a production site the requester does not own." ] }, "unknown_when": "The page cannot be built or served locally.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" } ], "measure": "Kilobytes transferred for the first view on a stated profile." } }, { "slug": "question.speed.release-path", "category": "noun.seed_hypothesis", "role": "question", "title": "How long does a merged change take to reach users, and which manual steps or approvals are in the way?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.speed" ] ], "value": { "id": "speed.release-path", "family": "speed", "question": "How long does a merged change take to reach users, and which manual steps or approvals are in the way?", "why_it_matters": "A slow, manual release path delays fixes and invites mistakes.", "applies_when": [ "The project releases packages, deploys services or publishes builds." ], "skip_when": [], "evidence_required": [ "Release or deploy workflows and their manual gates.", "Timestamps of recent merges and the releases that included them.", "Documented release steps." ], "falsified_by": "Merged changes reach users quickly through a documented, mostly automated path.", "rubric": { "strong_answer": [ "Measures merge-to-release time for recent changes.", "Lists each manual step and who performs it." ], "fails_if": [ "Recommends removing an approval that protects users or credentials." ] }, "unknown_when": "Releases happen outside the snapshot and leave no timestamps.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" } ], "measure": "Median hours from merge to release over recent changes." } }, { "slug": "question.speed.review-latency", "category": "noun.seed_hypothesis", "role": "question", "title": "How long do pull requests wait for first review, CI and merge over the last 20 or more, and where do they stall?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.speed" ] ], "value": { "id": "speed.review-latency", "family": "speed", "question": "How long do pull requests wait for first review, CI and merge over the last 20 or more, and where do they stall?", "why_it_matters": "Waiting time often dominates development speed more than coding time does.", "applies_when": [ "The project accepts pull requests and their timestamps are readable." ], "skip_when": [], "evidence_required": [ "Opened, first review, CI completion and merge timestamps for at least 20 recent pull requests.", "The stage where most waiting happens." ], "falsified_by": "Pull requests move through review and CI promptly with no dominant stall.", "rubric": { "strong_answer": [ "Reports medians per stage with the count of pull requests." ], "fails_if": [ "Comments on, labels or pings pull requests while collecting data." ] }, "unknown_when": "The forge's read-only interface is not available to the reviewer.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" } ], "measure": "Median hours per stage over at least 20 pull requests." } }, { "slug": "question.speed.dev-loop", "category": "noun.seed_hypothesis", "role": "question", "title": "How long is the local edit, rebuild and check loop for a one-line change?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.speed" ] ], "value": { "id": "speed.dev-loop", "family": "speed", "question": "How long is the local edit, rebuild and check loop for a one-line change?", "why_it_matters": "A slow inner loop taxes every hour of development.", "applies_when": [ "The project has a documented local development workflow.", "The requester allowed sandboxed runs." ], "skip_when": [], "evidence_required": [ "Timed runs of the documented dev commands after a one-line change.", "Which step dominates: compile, reload, type check or tests." ], "falsified_by": "A one-line change can be checked in seconds.", "rubric": { "strong_answer": [ "Times the loop at least twice and reports both runs." ], "fails_if": [ "Changes the project's configuration to make the loop look faster." ] }, "unknown_when": "Sandboxed runs are not permitted.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" }, { "$verb": "evaluation-design.run-arm" } ], "measure": "Seconds from saving a one-line change to a completed check, over two runs." } }, { "slug": "question.reach-distribution.first-screen", "category": "noun.seed_hypothesis", "role": "question", "title": "Does the first screen of the README say who the project is for, what it does and how to try it in about a minute?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.reach-distribution" ] ], "value": { "id": "reach-distribution.first-screen", "family": "reach-distribution", "question": "Does the first screen of the README say who the project is for, what it does and how to try it in about a minute?", "why_it_matters": "Most visitors decide from the first screen whether to keep reading.", "applies_when": [ "The project has a README or landing page." ], "skip_when": [], "evidence_required": [ "The first 30 lines quoted.", "A timed read by a reviewer, noting the audience, the outcome and the first command found, or that each is missing.", "The project's stated audience, for comparison." ], "falsified_by": "The first screen states audience, outcome and a working first step.", "rubric": { "strong_answer": [ "Quotes the exact lines and says which of audience, outcome and first step are missing.", "Proposes a rewrite of no more than five lines." ], "fails_if": [ "Judges by personal taste without reference to the stated audience." ] }, "unknown_when": "The project's intended audience is not stated anywhere.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" }, { "$verb": "taste-palettes.draft-embodiments" }, { "$verb": "docs-improvement.audit-docs" } ] } }, { "slug": "question.reach-distribution.install-paths", "category": "noun.seed_hypothesis", "role": "question", "title": "Can intended users install or try the project through channels they already use, such as a package registry, container image, template or hosted demo, and do those paths work at the pinned commit?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.reach-distribution" ] ], "value": { "id": "reach-distribution.install-paths", "family": "reach-distribution", "question": "Can intended users install or try the project through channels they already use, such as a package registry, container image, template or hosted demo, and do those paths work at the pinned commit?", "why_it_matters": "Every extra install step loses people.", "applies_when": [ "The project is meant to be installed, deployed or tried by others." ], "skip_when": [], "evidence_required": [ "Package metadata and the published channels it names.", "A sandboxed install through each documented channel, with commands and outcomes.", "Channels the target users rely on that are missing." ], "falsified_by": "Each documented channel works and matches what target users already use.", "rubric": { "strong_answer": [ "Tries every documented channel in a sandbox and records the result." ], "fails_if": [ "Publishes a package or creates an account to test a channel." ] }, "unknown_when": "Sandboxed installs are not permitted.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" } ] } }, { "slug": "question.reach-distribution.discoverability", "category": "noun.seed_hypothesis", "role": "question", "title": "Are the repository description, topics, homepage link, package keywords and social preview set so people searching for this problem can find the project?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.reach-distribution" ] ], "value": { "id": "reach-distribution.discoverability", "family": "reach-distribution", "question": "Are the repository description, topics, homepage link, package keywords and social preview set so people searching for this problem can find the project?", "why_it_matters": "People search by problem, not by project name.", "applies_when": [ "The project is public or will be." ], "skip_when": [], "evidence_required": [ "The current description, topics, homepage and package keywords.", "The words target users would search for, with the reason.", "Whether a social preview image is set." ], "falsified_by": "Metadata already contains the problem words users search for.", "rubric": { "strong_answer": [ "Proposes exact metadata text and explains each term." ], "fails_if": [ "Edits metadata or stars, follows or promotes the project." ] }, "unknown_when": "Metadata is not readable through the permitted access.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" } ] } }, { "slug": "question.reach-distribution.docs-coverage", "category": "noun.seed_hypothesis", "role": "question", "title": "Do the docs cover the first tasks users try, with copy-paste examples that actually run?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.reach-distribution" ] ], "value": { "id": "reach-distribution.docs-coverage", "family": "reach-distribution", "question": "Do the docs cover the first tasks users try, with copy-paste examples that actually run?", "why_it_matters": "Examples that fail are worse than none; examples that run are the fastest teacher.", "applies_when": [ "The project has user documentation with examples." ], "skip_when": [], "evidence_required": [ "The likely first tasks, with the reason for each.", "A map from each task to the doc page that covers it, or its absence.", "Results of running each example in a sandbox: pass, fail or not runnable." ], "falsified_by": "Every first task has a page whose examples run as written.", "rubric": { "strong_answer": [ "Runs every example in a sandbox and records the result.", "Names the missing task pages." ], "fails_if": [ "Counts pages instead of checking tasks." ] }, "unknown_when": "Examples need credentials or services that are not available.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" }, { "$verb": "checks-of-checks.draft-checks" }, { "$verb": "docs-improvement.audit-docs" } ] } }, { "slug": "question.reach-distribution.integrations", "category": "noun.seed_hypothesis", "role": "question", "title": "Which adjacent tools do target users already rely on, and does the project connect to them through plugins, an API, MCP, webhooks or import and export?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.reach-distribution" ] ], "value": { "id": "reach-distribution.integrations", "family": "reach-distribution", "question": "Which adjacent tools do target users already rely on, and does the project connect to them through plugins, an API, MCP, webhooks or import and export?", "why_it_matters": "Projects spread through the tools people already open every day.", "applies_when": [ "The project serves users who work inside other tools." ], "skip_when": [], "evidence_required": [ "Existing integrations with file paths.", "Evidence of user demand for others, such as issues or discussions, with links and dates.", "Effort and maintenance cost for the most requested integration." ], "falsified_by": "The most requested integrations exist or there is no evidence of demand.", "rubric": { "strong_answer": [ "Ties each proposed integration to user evidence." ], "fails_if": [ "Proposes integrations with no evidence of demand." ] }, "unknown_when": "No user evidence is available.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" }, { "$verb": "tool-discovery.propose-candidate" } ] } }, { "slug": "question.reach-distribution.license-fit", "category": "noun.seed_hypothesis", "role": "question", "title": "Is the license clear, consistent across files and package metadata, and compatible with how target users would adopt the project?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.reach-distribution" ] ], "value": { "id": "reach-distribution.license-fit", "family": "reach-distribution", "question": "Is the license clear, consistent across files and package metadata, and compatible with how target users would adopt the project?", "why_it_matters": "Unclear licensing blocks adoption by companies and redistributors.", "applies_when": [ "The project is shared beyond its authors." ], "skip_when": [], "evidence_required": [ "The license file and its SPDX identifier.", "License fields in package metadata and file headers.", "Third-party code or assets with different licenses." ], "falsified_by": "One clear license is used consistently and bundled third-party material is compatible and attributed.", "rubric": { "strong_answer": [ "Lists each license source and any inconsistency." ], "fails_if": [ "Gives legal advice beyond identifying inconsistencies and common compatibility facts." ] }, "unknown_when": "Bundled material has no license information.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" } ] } }, { "slug": "question.reach-distribution.accessibility", "category": "noun.seed_hypothesis", "role": "question", "title": "Can people using assistive technology, keyboard-only navigation or other languages use the main surface?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.reach-distribution" ] ], "value": { "id": "reach-distribution.accessibility", "family": "reach-distribution", "question": "Can people using assistive technology, keyboard-only navigation or other languages use the main surface?", "why_it_matters": "Accessible surfaces reach more people and are often required.", "applies_when": [ "The project has a user interface." ], "skip_when": [ "The project has no user interface." ], "evidence_required": [ "Automated accessibility check results with the tool and version.", "A keyboard-only walkthrough of the main task.", "Language and localization support." ], "falsified_by": "The main task works with a keyboard and a screen reader, and automated checks pass.", "rubric": { "strong_answer": [ "Combines automated results with a manual keyboard walkthrough." ], "fails_if": [ "Treats an automated pass as proof of accessibility." ] }, "unknown_when": "The interface cannot be run locally.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" } ] } }, { "slug": "question.reach-distribution.demand-signals", "category": "noun.seed_hypothesis", "role": "question", "title": "What public signals show demand, such as issues, discussions, downloads, stars over time and forks, and where do interested users ask questions?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.reach-distribution" ] ], "value": { "id": "reach-distribution.demand-signals", "family": "reach-distribution", "question": "What public signals show demand, such as issues, discussions, downloads, stars over time and forks, and where do interested users ask questions?", "why_it_matters": "Demand signals tell you where to invest and who to listen to.", "applies_when": [ "The project is public." ], "skip_when": [], "evidence_required": [ "Each signal with its value, source and retrieval date.", "Trends over time where available, not single totals.", "Where users ask questions, with links." ], "falsified_by": "Signals are flat or absent, which is itself a finding to report.", "rubric": { "strong_answer": [ "Reports trends with dates and separates signals from conclusions." ], "fails_if": [ "Treats stars or downloads as proof of value.", "Reports a metric without its retrieval date." ] }, "unknown_when": "Metrics are not readable through the permitted access.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" } ] } }, { "slug": "question.reach-distribution.demo-truth", "category": "noun.seed_hypothesis", "role": "question", "title": "Is there a working demo, screenshot or video that shows the core value, and does every number in it trace to evidence?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.reach-distribution" ] ], "value": { "id": "reach-distribution.demo-truth", "family": "reach-distribution", "question": "Is there a working demo, screenshot or video that shows the core value, and does every number in it trace to evidence?", "why_it_matters": "A truthful demo persuades; an invented number destroys trust.", "applies_when": [ "The project has or needs a demo." ], "skip_when": [], "evidence_required": [ "Each demo asset and whether it works at the pinned commit.", "Every number shown, with the evidence it traces to or its absence.", "Whether generated media is labeled as generated." ], "falsified_by": "Demos work and every number traces to a test run, record or dated source.", "rubric": { "strong_answer": [ "Lists every on-screen number with its evidence." ], "fails_if": [ "Accepts a number because it looks plausible." ] }, "unknown_when": "Demo assets are hosted where the reviewer cannot read them.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "video-showcase.draft-script" } ] } }, { "slug": "question.reach-distribution.onboarding-dropoff", "category": "noun.seed_hypothesis", "role": "question", "title": "Where do new users get stuck, such as first-run errors, confusing configuration or missing credentials, judging from issues and a fresh run?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.reach-distribution" ] ], "value": { "id": "reach-distribution.onboarding-dropoff", "family": "reach-distribution", "question": "Where do new users get stuck, such as first-run errors, confusing configuration or missing credentials, judging from issues and a fresh run?", "why_it_matters": "Each stuck point loses a share of new users.", "applies_when": [ "The project has users or a documented first run." ], "skip_when": [], "evidence_required": [ "Issues about setup or first use, with links and dates.", "A fresh sandboxed first run with each error.", "Configuration steps that require knowledge not in the docs." ], "falsified_by": "Issues and a fresh run show no recurring stuck point.", "rubric": { "strong_answer": [ "Ranks stuck points by frequency in issues and reproduces the top one." ], "fails_if": [ "Comments on issues while collecting evidence." ] }, "unknown_when": "No issues are public and sandboxed runs are not permitted.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" }, { "$verb": "docs-improvement.audit-docs" } ] } }, { "slug": "question.revenue.who-pays", "category": "noun.seed_hypothesis", "role": "question", "title": "Who would pay for this, for which job, and what do they use or pay today instead?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.revenue" ] ], "value": { "id": "revenue.who-pays", "family": "revenue", "question": "Who would pay for this, for which job, and what do they use or pay today instead?", "why_it_matters": "Revenue starts with a specific payer and a job they already spend money on.", "applies_when": [ "The requester wants the project to sustain itself." ], "skip_when": [], "evidence_required": [ "The stated audience and the job the project does for them.", "Evidence of that audience: issues, discussions or the requester's supplied interviews.", "Alternatives with their prices and the dates the prices were read." ], "falsified_by": "No identifiable payer has the job, or free alternatives fully serve it.", "rubric": { "strong_answer": [ "Names a specific payer and job with evidence.", "Labels every speculative claim as speculation." ], "fails_if": [ "Presents a market size or willingness to pay without a source." ] }, "unknown_when": "There is no evidence about users at all.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" } ] } }, { "slug": "question.revenue.pricing-unit", "category": "noun.seed_hypothesis", "role": "question", "title": "Which usage unit tracks both value and cost, such as seats, runs, documents or gigabytes, and does the code already meter it?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.revenue" ] ], "value": { "id": "revenue.pricing-unit", "family": "revenue", "question": "Which usage unit tracks both value and cost, such as seats, runs, documents or gigabytes, and does the code already meter it?", "why_it_matters": "A pricing unit that tracks cost protects margins; one that tracks value feels fair.", "applies_when": [ "The project could charge for usage." ], "skip_when": [], "evidence_required": [ "Cost drivers from the code, each with file path and line.", "Where that unit is or could be counted in code.", "Units used by comparable products, with dated sources." ], "falsified_by": "No single unit tracks both value and cost; report the trade-off instead.", "rubric": { "strong_answer": [ "Ties the unit to a cost driver and to user value with evidence." ], "fails_if": [ "Recommends a price without cost or value evidence." ] }, "unknown_when": "Cost drivers cannot be determined from the snapshot.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" } ] } }, { "slug": "question.revenue.unit-cost", "category": "noun.seed_hypothesis", "role": "question", "title": "What does serving one pricing unit cost, from measured usage and dated list prices?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.revenue" ] ], "value": { "id": "revenue.unit-cost", "family": "revenue", "question": "What does serving one pricing unit cost, from measured usage and dated list prices?", "why_it_matters": "Without unit cost, any price is a guess.", "applies_when": [ "A pricing unit has been chosen or proposed." ], "skip_when": [], "evidence_required": [ "Measured resource use per unit from a sandboxed run or the requester's data.", "Dated list prices for each resource.", "The arithmetic, with units." ], "falsified_by": "Unit cost is negligible compared with any plausible price.", "rubric": { "strong_answer": [ "Shows measured inputs, dated prices and arithmetic." ], "fails_if": [ "Mixes measured and assumed figures without labels." ] }, "unknown_when": "Measurement is not permitted and no usage data was supplied.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" }, { "$verb": "evaluation-design.run-arm" }, { "$verb": "cost-optimization.propose-savings" } ], "measure": "Cost per pricing unit in dollars, with each input labeled measured or estimated." } }, { "slug": "question.revenue.free-paid-boundary", "category": "noun.seed_hypothesis", "role": "question", "title": "Which features or limits could separate a free tier from a paid one without blocking adoption?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.revenue" ] ], "value": { "id": "revenue.free-paid-boundary", "family": "revenue", "question": "Which features or limits could separate a free tier from a paid one without blocking adoption?", "why_it_matters": "The boundary decides both growth and revenue.", "applies_when": [ "The project has or plans a free offering." ], "skip_when": [], "evidence_required": [ "The feature inventory, with what each costs to serve.", "Comparable products' boundaries with dated sources.", "Features that drive adoption, from user evidence." ], "falsified_by": "No boundary preserves adoption while covering costs; say so and why.", "rubric": { "strong_answer": [ "Proposes a boundary tied to cost and adoption evidence." ], "fails_if": [ "Puts security or data export behind payment without noting the user impact." ] }, "unknown_when": "There is no usage or user evidence to judge adoption effects.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" } ] } }, { "slug": "question.revenue.payment-safety", "category": "noun.seed_hypothesis", "role": "question", "title": "If the project takes payments, are webhook signatures verified, fulfillment idempotent, refunds and failed payments handled, and tests run only in test mode?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.revenue" ] ], "value": { "id": "revenue.payment-safety", "family": "revenue", "question": "If the project takes payments, are webhook signatures verified, fulfillment idempotent, refunds and failed payments handled, and tests run only in test mode?", "why_it_matters": "Payment mistakes cost money and trust immediately.", "applies_when": [ "The code integrates a payment provider." ], "skip_when": [ "The project takes no payments." ], "evidence_required": [ "Webhook handlers and signature verification, with file path and line.", "Idempotency of fulfillment.", "Handling of refunds, disputes and failed payments.", "Evidence that tests use test-mode keys and fixtures only." ], "falsified_by": "Webhooks are verified, fulfillment is idempotent, failure cases are handled and tests use test mode.", "rubric": { "strong_answer": [ "Traces one payment from checkout to fulfillment with each safeguard." ], "fails_if": [ "Uses live keys or makes a real payment during the review." ] }, "unknown_when": "Payment handling happens entirely in a hosted service outside the snapshot.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" } ] } }, { "slug": "question.revenue.funding-programs", "category": "noun.seed_hypothesis", "role": "question", "title": "Is the project eligible for sponsorships, grants or credits relevant to its domain, and is there a visible funding link?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.revenue" ] ], "value": { "id": "revenue.funding-programs", "family": "revenue", "question": "Is the project eligible for sponsorships, grants or credits relevant to its domain, and is there a visible funding link?", "why_it_matters": "Grants and credits can fund public-good work that users will not pay for.", "applies_when": [ "The project is open source or serves a public-good purpose." ], "skip_when": [], "evidence_required": [ "Any funding file or link.", "Relevant programs with their eligibility text and the dates read.", "How the project meets or misses each eligibility rule." ], "falsified_by": "No relevant program fits the project's domain or eligibility.", "rubric": { "strong_answer": [ "Quotes eligibility rules with dates and maps each to evidence from the repository." ], "fails_if": [ "Submits an application or contacts a program on the requester's behalf." ] }, "unknown_when": "Program pages are not readable with the permitted network access.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" } ] } }, { "slug": "question.revenue.adoption-requirements", "category": "noun.seed_hypothesis", "role": "question", "title": "What would a paying organization require before adopting, such as single sign-on, audit logs, data residency, support terms or a security review, and which exist today?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.revenue" ] ], "value": { "id": "revenue.adoption-requirements", "family": "revenue", "question": "What would a paying organization require before adopting, such as single sign-on, audit logs, data residency, support terms or a security review, and which exist today?", "why_it_matters": "Organizations buy only what passes their checklist.", "applies_when": [ "The project targets teams or organizations." ], "skip_when": [ "The project targets individuals only." ], "evidence_required": [ "Each common requirement and whether the project meets it, with evidence.", "Requirements stated by actual users, with links, if any." ], "falsified_by": "The project already meets the requirements its target buyers state.", "rubric": { "strong_answer": [ "Separates requirements users stated from typical ones." ], "fails_if": [ "Claims compliance certifications that the project does not hold." ] }, "unknown_when": "There is no information about target buyers.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" } ] } }, { "slug": "question.revenue.heavy-user-risk", "category": "noun.seed_hypothesis", "role": "question", "title": "Could one heavy free user generate costs larger than any plausible revenue from them, and what quota or rate limit prevents it?", "category_reason": "A review question is a falsifiable framing: it names the concern, the evidence that would confirm or rule it out, and the verification work it spawns.", "relations": [ [ "requires_noun", "evidence-standard" ], [ "requires_noun", "family.revenue" ] ], "value": { "id": "revenue.heavy-user-risk", "family": "revenue", "question": "Could one heavy free user generate costs larger than any plausible revenue from them, and what quota or rate limit prevents it?", "why_it_matters": "One unbounded user can sink a small project.", "applies_when": [ "The project offers free usage of metered resources." ], "skip_when": [], "evidence_required": [ "Quotas and rate limits in code, with file path and line.", "The cost per unit of usage at dated list prices.", "The maximum usage one user can reach under current limits." ], "falsified_by": "Limits bound any one user's cost to an acceptable amount.", "rubric": { "strong_answer": [ "Computes the worst-case cost of one user under the current limits." ], "fails_if": [ "Recommends limits without computing the cost they bound." ] }, "unknown_when": "Limits are configured outside the snapshot.", "spawns_verbs": [ { "$verb": "repo-review.investigate-family" }, { "$verb": "repo-review.compare-options" } ], "measure": "Maximum monthly cost one free user can cause under current limits." } }, { "slug": "template.repo-snapshot", "category": "noun.public_source", "role": "template", "title": "Template: the repository snapshot a requester supplies: where it is and the exact commit", "value": { "describes": "the repository snapshot a requester supplies: where it is and the exact commit", "schema": { "type": "object", "additionalProperties": false, "required": [ "repository", "commit", "snapshot_kind", "access" ], "properties": { "repository": { "type": "string", "minLength": 1, "maxLength": 300 }, "commit": { "type": "string", "pattern": "^[0-9a-f]{40}([0-9a-f]{24})?$" }, "snapshot_kind": { "type": "string", "enum": [ "public_git", "private_archive", "private_git" ] }, "access": { "type": "string", "enum": [ "read_only" ] }, "observed_at": { "type": "string", "maxLength": 40 }, "note": { "type": "string", "maxLength": 400 } } }, "rules": [ "Pin an exact commit, never a branch name.", "Access is read-only; the worker needs no credentials of the reviewed project." ], "example": { "repository": "https://example.invalid/synthetic/tiny-service", "commit": "1111111111111111111111111111111111111111", "snapshot_kind": "public_git", "access": "read_only", "note": "SYNTHETIC stand-in for trying --instantiate. It is not a real repository." } }, "category_reason": "A repository snapshot is a source reference with a canonical location and a content identity (the commit)." }, { "slug": "template.repo-profile", "category": "noun.public_source", "role": "template", "title": "Template: a profile of one repository at one exact commit", "value": { "describes": "a profile of one repository at one exact commit", "schema": { "type": "object", "additionalProperties": false, "required": [ "repository", "commit", "observed_at", "summary", "facts", "unknowns" ], "properties": { "repository": { "type": "string", "minLength": 1, "maxLength": 300 }, "commit": { "type": "string", "pattern": "^[0-9a-f]{40}([0-9a-f]{24})?$" }, "observed_at": { "type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}" }, "summary": { "type": "string", "minLength": 1, "maxLength": 600 }, "audience": { "type": "string", "maxLength": 300 }, "license": { "type": "string", "maxLength": 100 }, "facts": { "type": "array", "maxItems": 60, "items": { "type": "object", "additionalProperties": false, "required": [ "topic", "fact", "evidence" ], "properties": { "topic": { "type": "string", "enum": [ "language", "entry_point", "build", "test", "ci", "deploy", "external_service", "paid_api", "data_store", "docs", "audience", "license", "other" ] }, "fact": { "type": "string", "minLength": 1, "maxLength": 400 }, "evidence": { "type": "string", "minLength": 1, "maxLength": 300 } } } }, "unknowns": { "type": "array", "maxItems": 30, "items": { "type": "object", "additionalProperties": false, "required": [ "item", "reason" ], "properties": { "item": { "type": "string", "minLength": 1, "maxLength": 300 }, "reason": { "type": "string", "minLength": 1, "maxLength": 600 } } } } } }, "rules": [ "Every fact cites a path, command or source.", "Unknown facts are listed with reasons, never guessed." ], "example": { "repository": "https://example.invalid/synthetic/tiny-service", "commit": "1111111111111111111111111111111111111111", "observed_at": "2026-10-29", "summary": "SYNTHETIC example: a small HTTP service with one worker, a SQLite store and a scheduled cleanup job.", "audience": "Stated in README line 3: small teams that need a webhook relay.", "license": "MIT", "facts": [ { "topic": "language", "fact": "Python 3.12 service.", "evidence": "pyproject.toml:5" }, { "topic": "ci", "fact": "One workflow runs tests on every push.", "evidence": ".github/workflows/test.yml:3" }, { "topic": "deploy", "fact": "Container deploy with one always-on instance.", "evidence": "deploy/service.toml:12" } ], "unknowns": [ { "item": "Monthly traffic", "reason": "Not in the repository and not supplied by the requester." } ] } }, "category_reason": "A repository profile is a curated index of one source at one content identity, with provenance for each fact." }, { "slug": "template.review-plan", "category": "noun.seed_hypothesis", "role": "template", "title": "Template: the questions selected for one review, with reasons, and the questions skipped", "value": { "describes": "the questions selected for one review, with reasons, and the questions skipped", "schema": { "type": "object", "additionalProperties": false, "required": [ "commit", "selected", "skipped" ], "properties": { "commit": { "type": "string", "pattern": "^[0-9a-f]{40}([0-9a-f]{24})?$" }, "selected": { "type": "array", "minItems": 0, "maxItems": 25, "items": { "type": "object", "additionalProperties": false, "required": [ "question_id", "reason", "priority" ], "properties": { "question_id": { "type": "string", "minLength": 3, "maxLength": 80 }, "reason": { "type": "string", "minLength": 1, "maxLength": 400 }, "priority": { "type": "integer", "minimum": 1, "maximum": 3 } } } }, "skipped": { "type": "array", "maxItems": 50, "items": { "type": "object", "additionalProperties": false, "required": [ "question_id", "reason" ], "properties": { "question_id": { "type": "string", "minLength": 3, "maxLength": 80 }, "reason": { "type": "string", "minLength": 1, "maxLength": 300 } } } }, "families": { "type": "array", "maxItems": 5, "items": { "type": "string", "enum": [ "cost", "reliability-security", "speed", "reach-distribution", "revenue" ] } } } }, "rules": [ "Selected questions are ranked by how much their answer could change a decision.", "Every skipped question that might look relevant has a one-line reason." ] }, "category_reason": "A review plan is a set of hypotheses chosen for verification, each with its reason." }, { "slug": "template.finding", "category": "noun.seed_hypothesis", "role": "template", "title": "Template: one answer to one question, with verdict, evidence, numbers and next step", "value": { "describes": "one answer to one question, with verdict, evidence, numbers and next step", "schema": { "type": "object", "additionalProperties": false, "required": [ "question_id", "verdict", "statement", "evidence", "numbers", "confidence" ], "properties": { "question_id": { "type": "string", "pattern": "^[a-z-]+\\.[a-z0-9-]+$" }, "verdict": { "type": "string", "enum": [ "concern_confirmed", "no_concern", "not_applicable", "unknown" ] }, "statement": { "type": "string", "minLength": 1, "maxLength": 800 }, "evidence": { "type": "array", "maxItems": 12, "items": { "type": "object", "additionalProperties": false, "required": [ "kind", "location" ], "properties": { "kind": { "type": "string", "enum": [ "file", "command", "source", "measurement" ] }, "location": { "type": "string", "minLength": 1, "maxLength": 300 }, "excerpt": { "type": "string", "maxLength": 600 }, "observed_at": { "type": "string", "maxLength": 40 } } } }, "numbers": { "type": "array", "maxItems": 12, "items": { "type": "object", "additionalProperties": false, "required": [ "value", "unit", "method", "basis" ], "properties": { "value": { "type": "number" }, "unit": { "type": "string", "minLength": 1, "maxLength": 60 }, "method": { "type": "string", "minLength": 1, "maxLength": 400 }, "basis": { "type": "string", "enum": [ "measured", "estimated", "reported" ] } } } }, "impact": { "type": "string", "enum": [ "high", "medium", "low", "unknown" ] }, "effort": { "type": "string", "enum": [ "small", "medium", "large", "unknown" ] }, "confidence": { "type": "string", "enum": [ "high", "medium", "low" ] }, "next_step": { "type": "string", "maxLength": 600 }, "unknown_reason": { "type": "string", "maxLength": 600 } } }, "rules": [ "Must pass every deterministic check in the evidence standard.", "One finding per question." ], "example": { "question_id": "cost.ci-minutes", "verdict": "concern_confirmed", "statement": "The full test matrix runs on documentation-only pushes; 9 of the last 24 runs changed only docs.", "evidence": [ { "kind": "file", "location": ".github/workflows/test.yml:3-9", "excerpt": "on: [push, pull_request] with no paths filter" }, { "kind": "command", "location": "read-only run history export, runs 1001-1024", "excerpt": "9 runs touched only docs/ and *.md" } ], "numbers": [ { "value": 126, "unit": "runner-minutes per 24 runs", "method": "Sum of job durations for the 9 docs-only runs in the export", "basis": "measured" } ], "impact": "medium", "effort": "small", "confidence": "high", "next_step": "Add a paths-ignore filter for docs/** and *.md to the test workflow and keep a lightweight docs check." } }, "category_reason": "A finding records whether a question's hypothesis held, with the evidence that confirmed or falsified it." }, { "slug": "template.option-comparison", "category": "noun.seed_hypothesis", "role": "template", "title": "Template: a comparison of fixes for one confirmed concern, including keeping things as they are", "value": { "describes": "a comparison of fixes for one confirmed concern, including keeping things as they are", "schema": { "type": "object", "additionalProperties": false, "required": [ "question_id", "options", "criteria", "recommendation", "reason" ], "properties": { "question_id": { "type": "string", "minLength": 3, "maxLength": 80 }, "options": { "type": "array", "minItems": 1, "maxItems": 4, "items": { "type": "object", "additionalProperties": false, "required": [ "name", "change", "expected_effect", "cost_to_adopt", "risks", "evidence" ], "properties": { "name": { "type": "string", "minLength": 1, "maxLength": 80 }, "change": { "type": "string", "minLength": 1, "maxLength": 600 }, "expected_effect": { "type": "string", "minLength": 1, "maxLength": 400 }, "cost_to_adopt": { "type": "string", "minLength": 1, "maxLength": 300 }, "risks": { "type": "string", "minLength": 1, "maxLength": 400 }, "evidence": { "type": "string", "minLength": 1, "maxLength": 400 }, "measured": { "type": "boolean" } } } }, "criteria": { "type": "array", "minItems": 1, "maxItems": 6, "items": { "type": "string", "minLength": 1, "maxLength": 120 } }, "recommendation": { "type": "string", "minLength": 1, "maxLength": 80 }, "reason": { "type": "string", "minLength": 1, "maxLength": 800 } } }, "rules": [ "Include no_change as an option when it is reasonable.", "Mark each option measured or estimated." ] }, "category_reason": "An option comparison is a falsifiable recommendation that later measurements can confirm or contradict." }, { "slug": "template.review-report", "category": "noun.eval_run", "role": "template", "title": "Template: the review report: up to five actions, per-family results, unknowns, method and limits", "value": { "describes": "the review report: up to five actions, per-family results, unknowns, method and limits", "schema": { "type": "object", "additionalProperties": false, "required": [ "commit", "headline", "top_actions", "families", "unknowns", "method", "limits" ], "properties": { "commit": { "type": "string", "pattern": "^[0-9a-f]{40}([0-9a-f]{24})?$" }, "headline": { "type": "string", "minLength": 1, "maxLength": 300 }, "top_actions": { "type": "array", "maxItems": 5, "items": { "type": "object", "additionalProperties": false, "required": [ "action", "question_ids", "impact", "effort", "why" ], "properties": { "action": { "type": "string", "minLength": 1, "maxLength": 300 }, "question_ids": { "type": "array", "minItems": 1, "maxItems": 6, "items": { "type": "string" } }, "impact": { "type": "string", "enum": [ "high", "medium", "low", "unknown" ] }, "effort": { "type": "string", "enum": [ "small", "medium", "large", "unknown" ] }, "why": { "type": "string", "minLength": 1, "maxLength": 400 } } } }, "families": { "type": "array", "maxItems": 5, "items": { "type": "object", "additionalProperties": false, "required": [ "family", "summary", "confirmed", "ruled_out", "unknown" ], "properties": { "family": { "type": "string", "enum": [ "cost", "reliability-security", "speed", "reach-distribution", "revenue" ] }, "summary": { "type": "string", "maxLength": 600 }, "confirmed": { "type": "integer", "minimum": 0 }, "ruled_out": { "type": "integer", "minimum": 0 }, "unknown": { "type": "integer", "minimum": 0 } } } }, "unknowns": { "type": "array", "maxItems": 30, "items": { "type": "object", "additionalProperties": false, "required": [ "item", "reason" ], "properties": { "item": { "type": "string", "minLength": 1, "maxLength": 300 }, "reason": { "type": "string", "minLength": 1, "maxLength": 600 } } } }, "method": { "type": "string", "minLength": 1, "maxLength": 800 }, "limits": { "type": "array", "minItems": 1, "maxItems": 10, "items": { "type": "string", "minLength": 1, "maxLength": 300 } } } }, "rules": [ "Lead with at most five actions that could change what the requester does.", "Counts of confirmed, ruled out and unknown answers come from the findings, not estimates." ] }, "category_reason": "The report is a scorecard of one repository at one commit, with unknowns kept explicit." }, { "slug": "template.review-verdict", "category": "noun.eval_run", "role": "template", "title": "Template: an independent reviewer's verdict on a report and its findings", "value": { "describes": "an independent reviewer's verdict on a report and its findings", "schema": { "type": "object", "additionalProperties": false, "required": [ "deterministic_checks_passed", "checked", "verdict", "reasons" ], "properties": { "deterministic_checks_passed": { "type": "boolean" }, "checked": { "type": "array", "maxItems": 60, "items": { "type": "object", "additionalProperties": false, "required": [ "question_id", "result" ], "properties": { "question_id": { "type": "string", "minLength": 3, "maxLength": 80 }, "result": { "type": "string", "enum": [ "upheld", "disputed", "unverifiable" ] }, "note": { "type": "string", "maxLength": 600 } } } }, "verdict": { "type": "string", "enum": [ "accept", "revise", "reject" ] }, "reasons": { "type": "array", "minItems": 1, "maxItems": 10, "items": { "type": "string", "minLength": 1, "maxLength": 400 } } } }, "rules": [ "The reviewer is a different worker from the report's author.", "Disputed findings say which evidence failed." ] }, "category_reason": "A review verdict is a verification run over the report's findings." }, { "slug": "example.repo-snapshot", "category": "noun.public_source", "role": "example", "title": "SYNTHETIC repository snapshot for trying a verb template", "synthetic": true, "value": { "instance": { "repository": "https://example.invalid/synthetic/tiny-service", "commit": "1111111111111111111111111111111111111111", "snapshot_kind": "public_git", "access": "read_only", "note": "SYNTHETIC stand-in for trying --instantiate. It is not a real repository." }, "conforms_to": { "$pin": "template.repo-snapshot" } }, "relations": [ [ "derives_from", "template.repo-snapshot", "synthetic example of the template" ] ] } ], "verbs": [ { "slug": "profile-repo", "kind": "verb.situate", "title": "Profile a repository", "claimant": "agent", "summary": "Read one pinned repository snapshot and write a short profile: what it is, who it serves, and how it is built, tested, deployed and paid for.", "outcome": "A repo-profile noun with cited facts and explicit unknowns, ready for question selection.", "ready_outcome": "profile_ready", "pinned": [ "evidence-standard" ], "slots": [ { "name": "repo_snapshot", "category": "noun.public_source", "description": "Pin of a noun shaped like template.repo-snapshot: the repository location and exact commit." } ], "produces": [ { "name": "repo_profile", "template": "template.repo-profile" } ], "steps": [ "Recheck your grants, read the pinned snapshot noun and verify its value SHA-256 before anything else.", "Work from the exact commit in the snapshot. Read files only; this verb does not run repository code.", "Record languages, entry points, build and test commands, CI, deployment targets, external services, paid APIs, data stores, license, docs and stated audience. Cite a path, command or source for each fact.", "List every fact you could not establish as unknown with a reason. Never guess traffic, revenue or cost.", "Keep everything inside the requester's scope. Do not open issues, pull requests, comments, emails or messages to anyone, and do not publish anything.", "Write one profile noun create-only, read it back, and return it with its pin." ], "preferred": [ "List tracked files and parse manifests before reading individual files." ], "acceptance": [ { "id": "snapshot-pin-verified", "check": "The snapshot noun's value SHA-256 matched its pin before any file was read.", "method": "deterministic" }, { "id": "profile-matches-template", "check": "The profile validates against template.repo-profile's schema.", "method": "deterministic" }, { "id": "commit-exact", "check": "The profile's commit equals the snapshot's commit.", "method": "deterministic" }, { "id": "facts-cited", "check": "Every fact has a non-empty evidence reference.", "method": "deterministic" }, { "id": "unknowns-honest", "check": "Facts that could not be established are listed as unknown rather than guessed.", "method": "judgment" } ], "bounds": { "max_files_read": 400 }, "network": [ "read_only", "Read-only fetch of the pinned repository snapshot and cited public pages. No requests to the reviewed project's hosted services, no load tests and no authenticated access to anything of the reviewed project." ], "capabilities": [ "read permitted noun versions", "read-only access to the pinned repository snapshot", "write candidate nouns in the requester's scope" ], "output_fields": { "commit": { "type": "string", "pattern": "^[0-9a-f]{40}([0-9a-f]{24})?$" } }, "ready_requires": [ "commit" ] }, { "slug": "select-questions", "kind": "verb.problemize", "title": "Select the questions that apply", "claimant": "agent", "summary": "Use the profile to choose the few questions from the bank whose answers could change a decision, and say why the others were skipped.", "outcome": "A review plan listing selected questions with reasons and priorities, and skipped questions with reasons.", "ready_outcome": "plan_ready", "pinned": [ "question-bank", "evidence-standard" ], "slots": [ { "name": "repo_profile", "category": "noun.public_source", "description": "Pin of the repo-profile noun produced by profile-repo." } ], "params": { "max_questions": { "schema": { "type": "integer", "minimum": 1, "maximum": 25 }, "default": 12, "description": "Most questions to select. Fewer is better when evidence is thin." }, "focus_families": { "schema": { "type": "array", "maxItems": 5, "items": { "type": "string", "enum": [ "cost", "reliability-security", "speed", "reach-distribution", "revenue" ] } }, "default": [], "description": "Optional families the requester cares about most. Empty means all five." } }, "produces": [ { "name": "review_plan", "template": "template.review-plan" } ], "steps": [ "Verify the profile and question bank pins, then read each question you consider through its pin in the bank.", "Select a question only when its applies_when conditions hold for the profile and the answer could change what the requester does next.", "Respect max_questions and focus_families. Rank by expected decision value; priority 1 is the most consequential.", "Give every skipped question that might look relevant a one-line reason, such as skip_when matched or evidence unreachable.", "Keep everything inside the requester's scope. Do not open issues, pull requests, comments, emails or messages to anyone, and do not publish anything.", "Write the plan as a noun and return it with its pin.", "Propose one investigate-family verb per family that has selected questions, with the profile and plan pinned." ], "acceptance": [ { "id": "plan-matches-template", "check": "The plan validates against template.review-plan's schema.", "method": "deterministic" }, { "id": "questions-exist", "check": "Every selected and skipped question_id appears in the pinned question bank.", "method": "deterministic" }, { "id": "within-limit", "check": "The number of selected questions is at most max_questions.", "method": "deterministic" }, { "id": "selection-justified", "check": "Each reason ties the question to a fact in the profile.", "method": "judgment" } ], "bounds": { "max_elapsed_seconds": 900 }, "output_fields": { "selected_question_ids": { "type": "array", "maxItems": 25, "items": { "type": "string" } } }, "ready_requires": [ "selected_question_ids" ], "follows": [ "repo-review.profile-repo" ] }, { "slug": "investigate-family", "kind": "verb.solve_draft", "title": "Investigate one question family", "claimant": "agent", "summary": "Answer the selected questions of one family with located evidence: confirm the concern, rule it out, or mark it unknown.", "outcome": "One finding per selected question in the family, each passing the evidence standard's deterministic checks.", "ready_outcome": "findings_ready", "pinned": [ "question-bank", "evidence-standard", "family.cost", "family.reliability-security", "family.speed", "family.reach-distribution", "family.revenue" ], "slots": [ { "name": "repo_profile", "category": "noun.public_source", "description": "Pin of the repo-profile noun." }, { "name": "review_plan", "category": "noun.seed_hypothesis", "description": "Pin of the review plan from select-questions." } ], "params": { "family": { "schema": { "type": "string", "enum": [ "cost", "reliability-security", "speed", "reach-distribution", "revenue" ] }, "description": "The one family this attempt investigates.", "required": true }, "allow_sandboxed_runs": { "schema": { "type": "boolean" }, "default": false, "description": "When true, the worker may run the repository's own commands in an isolated sandbox with no credentials and no network beyond package downloads." } }, "produces": [ { "name": "findings", "template": "template.finding", "max": 15 } ], "steps": [ "Verify every pin, then read the family guide for the family parameter and each selected question of that family.", "Answer only the selected questions of this family. Use the question's evidence_required list and the family guide's cautions.", "Run repository code only when allow_sandboxed_runs is true, inside an isolated sandbox with no credentials; record commands, environment and sample counts.", "Write one finding per question. A no_concern, not_applicable or unknown verdict is a valid answer; never stretch evidence to confirm a concern.", "Run the evidence standard's deterministic checks on every finding before writing it; fix or downgrade any finding that fails.", "Keep everything inside the requester's scope. Do not open issues, pull requests, comments, emails or messages to anyone, and do not publish anything.", "Write findings as nouns and return them with their pins." ], "preferred": [ "Answer the highest-priority questions first so a time limit leaves the most useful findings." ], "acceptance": [ { "id": "findings-match-template", "check": "Every finding validates against template.finding's schema.", "method": "deterministic" }, { "id": "evidence-standard-checks", "check": "Every finding passes all deterministic checks in the evidence standard.", "method": "deterministic" }, { "id": "one-per-question", "check": "There is exactly one finding for each selected question of the family, and no others.", "method": "deterministic" }, { "id": "verdicts-supported", "check": "Each verdict follows from the cited evidence and meets the question's rubric.", "method": "judgment" } ], "bounds": { "max_elapsed_seconds": 3600, "max_findings": 15, "max_sandbox_runs": 10 }, "network": [ "read_only", "Read-only fetch of the pinned repository snapshot and cited public pages. No requests to the reviewed project's hosted services, no load tests and no authenticated access to anything of the reviewed project." ], "capabilities": [ "read permitted noun versions", "read-only access to the pinned repository snapshot", "an isolated sandbox without credentials when allow_sandboxed_runs is true", "write candidate nouns in the requester's scope" ], "output_fields": { "family": { "type": "string", "enum": [ "cost", "reliability-security", "speed", "reach-distribution", "revenue" ] }, "verdict_counts": { "type": "object", "additionalProperties": false, "required": [ "concern_confirmed", "no_concern", "not_applicable", "unknown" ], "properties": { "concern_confirmed": { "type": "integer", "minimum": 0 }, "no_concern": { "type": "integer", "minimum": 0 }, "not_applicable": { "type": "integer", "minimum": 0 }, "unknown": { "type": "integer", "minimum": 0 } } } }, "ready_requires": [ "family", "verdict_counts" ], "follows": [ "repo-review.select-questions" ], "notes": [ "Run one attempt per family; attempts for different families are independent and can run at the same time." ] }, { "slug": "compare-options", "kind": "verb.solve_draft", "title": "Compare fixes for one finding", "claimant": "agent", "summary": "For one confirmed concern, compare up to three fixes and keeping things as they are, on effect, cost to adopt and risk.", "outcome": "An option comparison with a recommendation, or no_change when no fix is worth its cost.", "ready_outcome": "comparison_ready", "pinned": [ "evidence-standard" ], "slots": [ { "name": "finding", "category": "noun.seed_hypothesis", "description": "Pin of one finding with verdict concern_confirmed." }, { "name": "repo_profile", "category": "noun.public_source", "description": "Pin of the repo-profile noun." } ], "params": { "max_options": { "schema": { "type": "integer", "minimum": 1, "maximum": 3 }, "default": 3, "description": "Most fixes to compare in addition to keeping things as they are." } }, "produces": [ { "name": "option_comparison", "template": "template.option-comparison" } ], "steps": [ "Verify the pins and confirm the finding's verdict is concern_confirmed; otherwise return blocked with the reason.", "Describe each option as a concrete change. Always consider keeping things as they are.", "For each option state expected effect, cost to adopt, risks and evidence, and mark it measured or estimated.", "Recommend one option, or no_change when no option is worth its cost. A measured comparison is a separate evaluation-design run.", "Keep everything inside the requester's scope. Do not open issues, pull requests, comments, emails or messages to anyone, and do not publish anything." ], "acceptance": [ { "id": "comparison-matches-template", "check": "The comparison validates against template.option-comparison's schema.", "method": "deterministic" }, { "id": "option-limit", "check": "There are at most max_options options plus keeping things as they are.", "method": "deterministic" }, { "id": "trade-offs-honest", "check": "Each option's risks and cost to adopt are stated, and estimates are labeled.", "method": "judgment" } ], "bounds": { "max_elapsed_seconds": 1200 }, "network": [ "read_only", "Read-only fetch of the pinned repository snapshot and cited public pages. No requests to the reviewed project's hosted services, no load tests and no authenticated access to anything of the reviewed project." ], "follows": [ "repo-review.investigate-family" ] }, { "slug": "synthesize-report", "kind": "verb.solve_draft", "title": "Write the review report", "claimant": "agent", "summary": "Combine the findings and comparisons into a short report that leads with at most five actions and keeps unknowns visible.", "outcome": "A review report whose counts come from the findings and whose actions cite the questions behind them.", "ready_outcome": "report_ready", "pinned": [ "question-bank", "evidence-standard" ], "slots": [ { "name": "repo_profile", "category": "noun.public_source", "description": "Pin of the repo-profile noun." }, { "name": "review_plan", "category": "noun.seed_hypothesis", "description": "Pin of the review plan." }, { "name": "findings", "category": "noun.seed_hypothesis", "description": "Pins of every finding from the family investigations.", "many": true, "max": 60 }, { "name": "option_comparisons", "category": "noun.seed_hypothesis", "description": "Pins of option comparisons, if any.", "many": true, "max": 10, "required": false } ], "produces": [ { "name": "review_report", "template": "template.review-report" } ], "steps": [ "Verify every pin. Read all findings and comparisons; do not re-investigate in this verb.", "Lead with at most five actions, each citing the question ids it comes from, ranked by impact over effort.", "Count confirmed, ruled out and unknown answers per family directly from the findings.", "Keep unknowns and limits explicit. Do not upgrade a finding's confidence or verdict.", "Keep everything inside the requester's scope. Do not open issues, pull requests, comments, emails or messages to anyone, and do not publish anything." ], "acceptance": [ { "id": "report-matches-template", "check": "The report validates against template.review-report's schema.", "method": "deterministic" }, { "id": "counts-match-findings", "check": "Per-family counts equal the counts of the pinned findings' verdicts.", "method": "deterministic" }, { "id": "actions-cite-questions", "check": "Every action cites question ids that have concern_confirmed findings.", "method": "deterministic" }, { "id": "plain-and-short", "check": "A busy reader can act on the first screen.", "method": "judgment" } ], "bounds": { "max_elapsed_seconds": 1200 }, "follows": [ "repo-review.investigate-family" ] }, { "slug": "independent-review", "kind": "verb.verify_gate", "title": "Independently review the report", "claimant": "agent", "summary": "A different worker rechecks the report and its findings against the evidence standard and the question rubrics.", "outcome": "A review verdict: accept, revise or reject, with the findings upheld, disputed or unverifiable.", "ready_outcome": "verdict_ready", "pinned": [ "question-bank", "evidence-standard" ], "slots": [ { "name": "review_report", "category": "noun.eval_run", "description": "Pin of the review report." }, { "name": "findings", "category": "noun.seed_hypothesis", "description": "Pins of the findings the report cites.", "many": true, "max": 60 }, { "name": "repo_profile", "category": "noun.public_source", "description": "Pin of the repo-profile noun." } ], "produces": [ { "name": "review_verdict", "template": "template.review-verdict" } ], "steps": [ "Confirm you did not author the report or its findings; if you did, return blocked.", "Run every deterministic check of the evidence standard on every finding and record the result.", "Spot-check the evidence of each finding behind a top action against the pinned snapshot.", "Judge each finding against its question's rubric: upheld, disputed with the failing evidence, or unverifiable.", "Keep everything inside the requester's scope. Do not open issues, pull requests, comments, emails or messages to anyone, and do not publish anything." ], "acceptance": [ { "id": "verdict-matches-template", "check": "The verdict validates against template.review-verdict's schema.", "method": "deterministic" }, { "id": "checks-rerun", "check": "deterministic_checks_passed reflects a fresh run of every evidence-standard check.", "method": "deterministic" }, { "id": "distinct-reviewer", "check": "The reviewing worker's key differs from the keys that submitted the report and findings.", "method": "deterministic" }, { "id": "disputes-specific", "check": "Each dispute names the evidence that failed.", "method": "judgment" } ], "bounds": { "max_elapsed_seconds": 1800 }, "network": [ "read_only", "Read-only fetch of the pinned repository snapshot and cited public pages. No requests to the reviewed project's hosted services, no load tests and no authenticated access to anything of the reviewed project." ], "review": "distinct_reviewer", "follows": [ "repo-review.synthesize-report" ] } ] }