{ "pack_id": "faiss-ivf-pq-and-ann-index-cascade", "title": "FAISS IVF-PQ and ANN index cascade", "version": "0.1.0", "summary": "A collection of nouns and verbs for k-means++ Voronoi inverted-file partitioning, residual uint8 product quantization, asymmetric distance lookup tables and exact top-R reranking.", "dependencies": [ "configuration-grids-and-bounds", "evaluation-design", "wiring-and-conformance-checks" ], "boundaries": [ "ProductQuantizedIvfIndex partitions R^D vectors across nlist Voronoi cells via deterministic k-means++, quantizes residual subvectors into compact uint8 PQ codes and evaluates queries via Asymmetric Distance Computation (ADC) lookup tables.", "Exact dense L2 reranking executes strictly on the shortlisted top-R ADC candidates when enabled by the active NodeGrid regime.", "All index topology, codebook, probing and rerank parameters resolve from default_grid_spec.json with zero inline magic literals.", "Importing the pack writes nothing, enqueues nothing and installs no code." ], "nouns": [ { "slug": "pack", "category": "noun.skill_pack", "role": "pack", "title": "A collection of nouns and verbs for k-means++ Voronoi inverted-file partitioning, residual uint8 product quantization, asymmetric distance lookup tables and exact top-R reranking", "value": { "lets_workers": "Fit deterministic k-means++ coarse Voronoi quantizers (nlist cells) and M residual product-quantization sub-codebooks (K_sub centroids per subspace), pack vectors into M-byte uint8 PQ codes, run Asymmetric Distance Computation (ADC) table lookups across nprobe cells with optional exact top-R reranking, and benchmark memory compression vs Recall@k across fast_compact and high_recall grid cells.", "flow": [ { "verb": { "$verb": "faiss-ivf-pq-and-ann-index-cascade.fit-ivf-pq-voronoi-and-residual-codebooks" }, "when": "Train coarse Voronoi centroids and M residual subspace codebooks via deterministic k-means++ and encode indexed vectors as compact M-byte uint8 PQ codes." }, { "verb": { "$verb": "faiss-ivf-pq-and-ann-index-cascade.benchmark-adc-and-exact-rerank-regimes" }, "when": "Evaluate nprobe Asymmetric Distance Computation lookup tables and exact top-R rerank cascades across fast_compact and high_recall grid cells against brute-force ground truth." }, { "verb": { "$verb": "faiss-ivf-pq-and-ann-index-cascade.verify-ivf-pq-compression-and-recall-receipt" }, "when": "Independently verify byte compression ratios, Recall@k progression and ZeroHardcodedValuesAstGate conformance." } ], "guardrails": [ { "$pin": "guardrail.divisible-subspaces-and-uint8-codebook-bounds" }, { "$pin": "guardrail.grid-governed-cascade-and-zero-magic-literals" } ], "tool": { "$pin": "tool.ivf-pq-cascade" }, "worked_example": "192-vector 32-D IVF-PQ cascade benchmark recomputed in test_ivf_pq_cascade.py: fast_compact packs 4 uint8 bytes/vector (64.0x compression vs float64, 32.0x vs float32, scanning 24.0 vectors/query with 0 exact reranks, Recall@5 = 0.533333), while high_recall packs 8 uint8 bytes/vector (32.0x vs float64, 16.0x vs float32, scanning 96.0 ADC candidates and reranking top 20, achieving Recall@5 = 1.0)." }, "relations": [ [ "requires_noun", "guardrail.divisible-subspaces-and-uint8-codebook-bounds" ], [ "requires_noun", "guardrail.grid-governed-cascade-and-zero-magic-literals" ], [ "requires_noun", "spec.ivf-pq-and-adc-cascade-grid" ], [ "requires_noun", "rules.residual-pq-and-adc-lookup" ], [ "requires_noun", "tool.ivf-pq-cascade" ], [ "requires_noun", "template.ivf-pq-cascade-receipt" ] ] }, { "slug": "guardrail.divisible-subspaces-and-uint8-codebook-bounds", "category": "noun.skill_pack", "role": "guardrail", "title": "Guardrail: vector dimensions must divide evenly across M subspaces and K_sub centroids must fit uint8 storage", "category_reason": "Preserves lossless subspace slicing and 1-byte-per-subspace PQ code layout.", "value": { "rule": "Require dim % subspaces_m == 0, 1 <= subspace_centroids_k <= max_uint8_centroids (256) and nprobe <= nlist.", "how_it_is_checked": "ProductQuantizedIvfIndex.fit and NodeGrid.is_applicable reject non-divisible dimensions, out-of-range codebooks and invalid probe counts." } }, { "slug": "guardrail.grid-governed-cascade-and-zero-magic-literals", "category": "noun.skill_pack", "role": "guardrail", "title": "Guardrail: all Voronoi, PQ codebook and top-R rerank scalars resolve from NodeGrid with zero inline magic literals", "category_reason": "Prevents hardcoded index constants and enforces cross-parameter rerank constraints.", "value": { "rule": "Resolve nlist, nprobe, subspaces_m, subspace_centroids_k and rerank_top_r from default_grid_spec.json and enforce ZeroHardcodedValuesAstGate.", "how_it_is_checked": "scan_directory_for_hardcoded_values and NodeGrid constraint checks run during make check." } }, { "slug": "spec.ivf-pq-and-adc-cascade-grid", "category": "noun.benchmark_suite", "role": "spec", "title": "Specification: ProductQuantizedIvfIndex Voronoi partitioning, uint8 PQ encoding and ADC + top-R cascade", "category_reason": "Defines coarse quantizer, residual product quantizer, ADC lookup table and exact rerank cascade parameters.", "value": { "components": [ "ProductQuantizedIvfIndex", "kmeans_plus_plus_fit", "benchmark_ivf_pq_regimes" ], "regimes": [ "fast_compact", "high_recall" ], "fast_compact_subspaces_m": 4, "high_recall_subspaces_m": 8, "max_uint8_centroids": 256 } }, { "slug": "rules.residual-pq-and-adc-lookup", "category": "noun.reflex_procedure", "role": "rules", "title": "Deterministic residual quantization, M x K_sub ADC lookup tables and top-R exact rerank equations", "category_reason": "Zero-model ANN indexing and asymmetric distance lookup equations.", "value": { "residual_equation": "r(x) = x - C_{c(x)}, partitioned into M contiguous subvectors r_1(x), ..., r_M(x) in R^{D/M}", "adc_lookup_equation": "d_ADC^2(q, x) = sum_{m=0}^{M-1} LUT_c[m][code_m(x)], where LUT_c[m][k] = ||(q - C_c)_m - U_{m,k}||_2^2", "compression_equation": "compression_vs_float64 = (D * 8) / M; compression_vs_float32 = (D * 4) / M", "model_calls": 0 } }, { "slug": "tool.ivf-pq-cascade", "category": "noun.code_node", "role": "tool_card", "title": "Tool card: standard-library IVF-PQ index with k-means++, uint8 PQ codes, ADC lookup and top-R reranking", "category_reason": "Tested Python 3.10+ standard-library package pinned by path, file count and SHA-256.", "value": { "name": "FAISS IVF-PQ and ANN index cascade", "code": { "$code": "faiss-ivf-pq-and-ann-index-cascade" }, "runtime": "Python 3.10 or later, standard library only.", "commands": [ "python -m ivf_pq_cascade --bundle BUNDLE.json" ], "does": [ "Implements deterministic k-means++ initialization and Lloyd iterations for coarse Voronoi cells and M residual PQ sub-codebooks.", "Stores indexed vectors as compact M-byte uint8 PQ codes inside inverted lists and answers queries via M x K_sub Asymmetric Distance Computation lookup tables plus optional exact top-R reranking.", "Benchmarks exact byte compression ratios vs float64/float32 and Recall@k across fast_compact and high_recall grid cells." ], "does_not": [ "Require C++ extensions, external numerical packages or hardcoded numeric scalars in operational functions." ], "code_license": "MIT" }, "relations": [ [ "verifies", "spec.ivf-pq-and-adc-cascade-grid" ] ] }, { "slug": "template.ivf-pq-cascade-receipt", "category": "noun.scene_recipe", "role": "template", "title": "Template: IVF-PQ memory compression and Recall@k regime benchmark receipt", "value": { "describes": "a deterministic receipt recording PQ byte footprints, compression ratios vs float64/float32, ADC candidate scan counts, exact rerank counts and Recall@k across fast_compact and high_recall regimes", "schema": { "type": "object", "additionalProperties": false, "required": [ "num_vectors", "dim", "num_queries", "top_k", "fast_compact_pq_bytes", "fast_compact_compression_vs_float64", "fast_compact_compression_vs_float32", "fast_compact_recall_at_k", "fast_compact_mean_adc_scanned", "fast_compact_mean_exact_reranked", "high_recall_pq_bytes", "high_recall_compression_vs_float64", "high_recall_compression_vs_float32", "high_recall_recall_at_k", "high_recall_mean_adc_scanned", "high_recall_mean_exact_reranked" ], "properties": { "num_vectors": { "type": "integer", "minimum": 1 }, "dim": { "type": "integer", "minimum": 1 }, "num_queries": { "type": "integer", "minimum": 1 }, "top_k": { "type": "integer", "minimum": 1 }, "fast_compact_pq_bytes": { "type": "integer", "minimum": 1 }, "fast_compact_compression_vs_float64": { "type": "number", "minimum": 1.0 }, "fast_compact_compression_vs_float32": { "type": "number", "minimum": 1.0 }, "fast_compact_recall_at_k": { "type": "number", "minimum": 0.0, "maximum": 1.0 }, "fast_compact_mean_adc_scanned": { "type": "number", "minimum": 0.0 }, "fast_compact_mean_exact_reranked": { "type": "number", "minimum": 0.0 }, "high_recall_pq_bytes": { "type": "integer", "minimum": 1 }, "high_recall_compression_vs_float64": { "type": "number", "minimum": 1.0 }, "high_recall_compression_vs_float32": { "type": "number", "minimum": 1.0 }, "high_recall_recall_at_k": { "type": "number", "minimum": 0.0, "maximum": 1.0 }, "high_recall_mean_adc_scanned": { "type": "number", "minimum": 0.0 }, "high_recall_mean_exact_reranked": { "type": "number", "minimum": 0.0 } } }, "example": { "num_vectors": 192, "dim": 32, "num_queries": 24, "top_k": 5, "fast_compact_pq_bytes": 4, "fast_compact_compression_vs_float64": 64.0, "fast_compact_compression_vs_float32": 32.0, "fast_compact_recall_at_k": 0.533333, "fast_compact_mean_adc_scanned": 24.0, "fast_compact_mean_exact_reranked": 0.0, "high_recall_pq_bytes": 8, "high_recall_compression_vs_float64": 32.0, "high_recall_compression_vs_float32": 16.0, "high_recall_recall_at_k": 1.0, "high_recall_mean_adc_scanned": 96.0, "high_recall_mean_exact_reranked": 20.0 } } }, { "slug": "example.ivf-pq-cascade-bundle", "category": "noun.eval_run", "role": "example", "synthetic": true, "title": "Worked example input: seeded 192-vector 32-D clustered corpus and fast_compact vs high_recall regime evaluation", "category_reason": "Synthetic fixture recomputed by test_ivf_pq_cascade.py.", "value": { "synthetic_corpus": { "num_vectors": 192, "dim": 32, "num_clusters": 8, "num_queries": 24, "center_scale": 5.0, "cluster_spread": 0.55, "query_noise": 0.45, "seed": 17 }, "top_k": 5, "regimes": [ "fast_compact", "high_recall" ], "shows": "IVF-PQ coarse Voronoi clustering, uint8 residual product quantization, ADC table lookups and exact top-R reranking.", "source": "Synthetic. Written for this pack." } }, { "slug": "example.ivf-pq-cascade-receipt", "category": "noun.eval_run", "role": "example", "synthetic": true, "title": "Worked example output: recomputed IVF-PQ compression and Recall@5 receipt", "category_reason": "Synthetic receipt verified by test_ivf_pq_cascade.py.", "value": { "conforms_to": { "$pin": "template.ivf-pq-cascade-receipt" }, "instance": { "num_vectors": 192, "dim": 32, "num_queries": 24, "top_k": 5, "fast_compact_pq_bytes": 4, "fast_compact_compression_vs_float64": 64.0, "fast_compact_compression_vs_float32": 32.0, "fast_compact_recall_at_k": 0.533333, "fast_compact_mean_adc_scanned": 24.0, "fast_compact_mean_exact_reranked": 0.0, "high_recall_pq_bytes": 8, "high_recall_compression_vs_float64": 32.0, "high_recall_compression_vs_float32": 16.0, "high_recall_recall_at_k": 1.0, "high_recall_mean_adc_scanned": 96.0, "high_recall_mean_exact_reranked": 20.0 }, "shows": "Recomputes 64.0x/32.0x float64/float32 compression in fast_compact and 1.00 Recall@5 with 32.0x/16.0x compression and top-20 reranking in high_recall.", "source": "Synthetic. Recomputed directly from example.ivf-pq-cascade-bundle." }, "relations": [ [ "derives_from", "example.ivf-pq-cascade-bundle" ], [ "verifies", "template.ivf-pq-cascade-receipt" ] ] } ], "verbs": [ { "slug": "fit-ivf-pq-voronoi-and-residual-codebooks", "kind": "verb.probe_source", "title": "Train deterministic k-means++ Voronoi cells and residual product-quantization codebooks", "claimant": "agent", "model_use": "none", "summary": "Partition dense vectors across nlist coarse centroids with k-means++, train M subspace residual codebooks of K_sub centroids and store compact M-byte uint8 PQ codes in inverted lists.", "outcome": "Fitted IVF-PQ index and exact byte compression ratios vs float64 and float32 vectors.", "ready_outcome": "ivf_pq_codebooks_fitted", "pinned": [ "spec.ivf-pq-and-adc-cascade-grid", "rules.residual-pq-and-adc-lookup", "guardrail.divisible-subspaces-and-uint8-codebook-bounds", "tool.ivf-pq-cascade" ], "slots": [ { "name": "vector_corpus_bundle", "category": "noun.eval_run", "description": "Pin of dense input vectors or deterministic synthetic cluster corpus parameters." } ], "produces": [ { "name": "index_compression_receipt", "template": "template.ivf-pq-cascade-receipt" } ], "steps": [ "Validate dimension divisibility by subspaces_m and uint8 centroid capacity from default_grid_spec.json.", "Fit coarse Voronoi centroids and M residual subspace codebooks with seeded k-means++ and pack uint8 PQ codes." ], "acceptance": [ { "id": "compact-uint8-pq-storage", "check": "Every indexed vector is stored as exactly M uint8 bytes achieving (D * 8) / M compression vs float64.", "method": "deterministic" } ] }, { "slug": "benchmark-adc-and-exact-rerank-regimes", "kind": "verb.generate_seed", "title": "Benchmark Asymmetric Distance Computation lookup tables and exact top-R reranking across grid regimes", "claimant": "agent", "model_use": "none", "summary": "Probe nprobe nearest Voronoi cells per query using M x K_sub Asymmetric Distance Computation lookup tables, optionally rerank top-R candidates with exact L2 distances and measure Recall@k against brute-force search.", "outcome": "Regime comparison scorecard covering compression ratios, scanned candidates, exact reranks and Recall@k.", "ready_outcome": "ivf_pq_regimes_benchmarked", "pinned": [ "spec.ivf-pq-and-adc-cascade-grid", "rules.residual-pq-and-adc-lookup", "guardrail.grid-governed-cascade-and-zero-magic-literals", "tool.ivf-pq-cascade" ], "slots": [ { "name": "query_benchmark_bundle", "category": "noun.eval_run", "description": "Pin of corpus, query vectors, top_k target and grid regime list." } ], "produces": [ { "name": "regime_benchmark_receipt", "template": "template.ivf-pq-cascade-receipt" } ], "steps": [ "Build M x K_sub ADC lookup tables per probed Voronoi cell and shortlist candidates using uint8 code lookups.", "Run exact L2 reranking on top-R shortlisted candidates when enabled and compute Recall@k vs brute-force k-NN." ], "acceptance": [ { "id": "high-recall-cascade-dominates-recall", "check": "high_recall regime achieves strictly higher Recall@k than fast_compact while scanning a fraction of dense vectors for exact L2 reranking.", "method": "deterministic" } ], "follows": [ "faiss-ivf-pq-and-ann-index-cascade.fit-ivf-pq-voronoi-and-residual-codebooks" ] }, { "slug": "verify-ivf-pq-compression-and-recall-receipt", "kind": "verb.verify_gate", "title": "Independently verify IVF-PQ byte compression, Recall@k progression and AST magic-literal gates", "claimant": "agent", "model_use": "none", "summary": "Recompute the worked IVF-PQ compression and Recall@k receipt from input specifications, verify schema conformance and run ZeroHardcodedValuesAstGate across ivf_pq_cascade.", "outcome": "Independent deterministic verification receipt.", "ready_outcome": "ivf_pq_receipt_verified", "review": "distinct_reviewer", "pinned": [ "spec.ivf-pq-and-adc-cascade-grid", "guardrail.divisible-subspaces-and-uint8-codebook-bounds", "guardrail.grid-governed-cascade-and-zero-magic-literals", "tool.ivf-pq-cascade", "template.ivf-pq-cascade-receipt" ], "slots": [ { "name": "ivf_pq_eval_bundle", "category": "noun.eval_run", "description": "Pin of IVF-PQ benchmark bundle and candidate receipt." } ], "produces": [ { "name": "verified_ivf_pq_receipt", "template": "template.ivf-pq-cascade-receipt" } ], "steps": [ "Run ivf_pq_cascade on the worked bundle and compare every field against template.ivf-pq-cascade-receipt.", "Run scan_directory_for_hardcoded_values across ivf_pq_cascade and confirm zero AST magic-literal violations." ], "acceptance": [ { "id": "ast-conformance-and-receipt-reproduction", "check": "scan_directory_for_hardcoded_values returns an empty list and run_bundle reproduces example.ivf-pq-cascade-receipt identically.", "method": "deterministic" } ], "follows": [ "faiss-ivf-pq-and-ann-index-cascade.benchmark-adc-and-exact-rerank-regimes" ] } ] }