Dria
Swan
NFTHardhat
21,000 USDC
View results
Previous Next
Submission Details
Severity: medium
Valid

Validation Order Dependency Vulnerability

abhishekthakur

Summary

The validation scoring system in LLMOracleCoordinator is vulnerable to order dependency where early validators can influence the scoring standard deviation, affecting reward distribution for subsequent validators.

Vulnerability Details

https://github.com/Cyfrin/2024-10-swan-dria/blob/main/contracts/llm/LLMOracleCoordinator.sol

In LLMOracleCoordinator, validation scoring is done in finalizeValidation:

function finalizeValidation(uint256 taskId) private {
TaskRequest storage task = requests[taskId];
​
// compute score for each generation
for (uint256 g_i = 0; g_i < task.parameters.numGenerations; g_i++) {
// get the scores for this generation
uint256[] memory scores = new uint256[]();
for (uint256 v_i = 0; v_i < task.parameters.numValidations; v_i++) {
scores[v_i] = validations[taskId][v_i].scores[g_i];
}
​
// compute the mean and standard deviation
(uint256 _stddev, uint256 _mean) = Statistics.stddev(scores);
​
// compute the score for this generation as the "inner-mean"
// and send rewards to validators that are within the range
uint256 innerSum = 0;
uint256 innerCount = 0;
for (uint256 v_i = 0; v_i < task.parameters.numValidations; ++v_i) {
uint256 score = scores[v_i];
if ((score >= _mean - _stddev) && (score <= _mean + _stddev)) {
innerSum += score;
innerCount++;
// send validation fee to the validator
_increaseAllowance(validations[taskId][v_i].validator, task.validatorFee);
}
}
}
}

The issues are:

  1. Standard deviation range is influenced by earlier validations

  2. Validators can see previous validations before submitting

  3. The reward distribution is based on being within mean ± stddev range

POC

// SPDX-License-Identifier: MIT
pragma solidity ^0.8.20;
​
import {Test} from "forge-std/Test.sol";
import {LLMOracleCoordinator} from "../contracts/llm/LLMOracleCoordinator.sol";
import {MockERC20} from "./mocks/MockERC20.sol";
​
contract ValidationOrderTest is Test {
LLMOracleCoordinator coordinator;
MockERC20 token;
address owner;
address[] validators;
uint256 constant NUM_VALIDATORS = 5;
uint256 constant VALIDATION_FEE = 1 ether;
function setUp() public {
owner = makeAddr("owner");
token = new MockERC20();
// Setup validators
validators = new address[](NUM_VALIDATORS);
for(uint i = 0; i < NUM_VALIDATORS; i++) {
validators[i] = makeAddr(string.concat("validator", vm.toString(i)));
token.mint(validators[i], 100 ether);
}
​
// Deploy coordinator with test parameters
coordinator = new LLMOracleCoordinator();
coordinator.initialize(
/* registry */ address(0),
address(token),
0, // platformFee
0, // generationFee
VALIDATION_FEE
);
}
​
function testValidationOrderManipulation() public {
// Create a task (simplified)
uint256 taskId = createTestTask();
// Scenario 1: Normal Distribution First
// Reset coordinator state
coordinator = new LLMOracleCoordinator();
uint256[] memory normalScores = new uint256[]();
normalScores[0] = 70;
normalScores[1] = 75;
normalScores[2] = 80;
// First 3 validators submit normal scores
for(uint i = 0; i < 3; i++) {
vm.prank(validators[i]);
coordinator.validate(taskId, 123, new uint256[](1){normalScores[i]}, "");
}
// Last 2 validators see the mean (~75) and stddev (~5)
// They can submit scores within this tight range
vm.prank(validators[3]);
coordinator.validate(taskId, 123, new uint256[](1){74}, "");
vm.prank(validators[4]);
coordinator.validate(taskId, 123, new uint256[](1){76}, "");
// Check rewards - all validators get rewarded as they're within ±stddev
for(uint i = 0; i < NUM_VALIDATORS; i++) {
assertEq(
coordinator.validatorRewards(validators[i]),
VALIDATION_FEE,
"Validator should receive reward"
);
}
​
// Scenario 2: Extreme Scores First
// Reset coordinator state
coordinator = new LLMOracleCoordinator();
taskId = createTestTask();
// First 3 validators submit extreme scores
vm.prank(validators[0]);
coordinator.validate(taskId, 123, new uint256[](1){10}, "");
vm.prank(validators[1]);
coordinator.validate(taskId, 123, new uint256[](1){90}, "");
vm.prank(validators[2]);
coordinator.validate(taskId, 123, new uint256[](1){50}, "");
// Last 2 validators now see mean=50, stddev=40
// They can submit any score between 10-90 and get rewarded
vm.prank(validators[3]);
coordinator.validate(taskId, 123, new uint256[](1){85}, "");
vm.prank(validators[4]);
coordinator.validate(taskId, 123, new uint256[](1){15}, "");
// Again, all validators get rewarded despite huge variance
for(uint i = 0; i < NUM_VALIDATORS; i++) {
assertEq(
coordinator.validatorRewards(validators[i]),
VALIDATION_FEE,
"Validator should receive reward"
);
}
}
function testTargetedExclusion() public {
uint256 taskId = createTestTask();
// First 4 validators collude to set a narrow range
for(uint i = 0; i < 4; i++) {
vm.prank(validators[i]);
coordinator.validate(taskId, 123, new uint256[](1){50}, "");
}
// Last validator submits honest but different score
vm.prank(validators[4]);
coordinator.validate(taskId, 123, new uint256[](1){75}, "");
// Check rewards - last validator gets excluded
for(uint i = 0; i < NUM_VALIDATORS - 1; i++) {
assertEq(
coordinator.validatorRewards(validators[i]),
VALIDATION_FEE,
"Colluding validator should receive reward"
);
}
assertEq(
coordinator.validatorRewards(validators[4]),
0,
"Honest validator should be excluded"
);
}
​
function createTestTask() internal returns (uint256) {
// Simplified task creation
bytes memory input = "test input";
bytes memory models = "test models";
LLMOracleTaskParameters memory params = LLMOracleTaskParameters({
difficulty: 1,
numGenerations: 1,
numValidations: NUM_VALIDATORS
});
return coordinator.request(bytes32("test"), input, models, params);
}
}

Impact

Financial Impact:

    • Early validators can manipulate reward distribution

  • Honest validators can be excluded by coordinated early submissions

  • Validation fees may be distributed unfairly

    Data Quality Impact:

    • Score manipulation affects purchase decisions

    • False consensus can be created

    • True outliers might be included in acceptable range

    Game Theory Impact:

    • Incentivizes validator collusion

    • Creates race condition for early validation

    • May lead to strategic validation timing

Tools Used

Manual Review, Foundry

Recommendations

Here are few recommendations:

Use Commit-Reveal Pattern:

contract LLMOracleCoordinator {
struct ValidationCommitment {
bytes32 commitment;
uint256 timestamp;
bool revealed;
}
mapping(uint256 => mapping(address => ValidationCommitment)) public commitments;
function commitValidation(uint256 taskId, bytes32 commitment) external {
commitments[taskId][msg.sender] = ValidationCommitment({
commitment: commitment,
timestamp: block.timestamp,
revealed: false
});
}
function revealValidation(
uint256 taskId,
uint256[] calldata scores,
bytes32 salt
) external {
ValidationCommitment storage commitment = commitments[taskId][msg.sender];
require(!commitment.revealed, "Already revealed");
require(
keccak256(abi.encodePacked(scores, salt)) == commitment.commitment,
"Invalid revelation"
);
commitment.revealed = true;
_processValidation(taskId, scores);
}
}

Use Median for Scoring:

function finalizeValidation(uint256 taskId) private {
uint256[] memory scores = new uint256[]();
for(uint256 i = 0; i < scores.length; i++) {
scores[i] = validations[taskId][i].scores[0];
}
uint256 median = Statistics.median(scores);
uint256 deviation = DEVIATION_THRESHOLD;
for(uint256 i = 0; i < scores.length; i++) {
if (abs(scores[i] - median) <= deviation) {
_increaseAllowance(validations[taskId][i].validator, task.validatorFee);
}
}
}

Implement Fixed Validation Windows:

struct TaskValidationWindow {
uint256 startTime;
uint256 endTime;
bool finalized;
}
​
mapping(uint256 => TaskValidationWindow) public validationWindows;
​
function validate(uint256 taskId, uint256[] calldata scores) external {
TaskValidationWindow storage window = validationWindows[taskId];
require(
block.timestamp >= window.startTime &&
block.timestamp <= window.endTime,
"Outside validation window"
);
// Store validation
validations[taskId].push(Validation({
validator: msg.sender,
scores: scores,
timestamp: block.timestamp
}));
}
Updates

Lead Judging Commences

inallhonesty Lead Judge 12 months ago
Submission Judgement Published
Validated
Assigned finding tags:

Unbounded score values in `validate` function

Support

FAQs

Can't find an answer? Chat with us on Discord, Twitter or Linkedin.