这个文件主要是在用验证集的分数去评判测试集，避免了数据泄漏，但是还不是针对单个病例具体分析，因为相当于还在抄验证集里age+location那个最好的策略¶

以下回答来自ChatGPT:现在为什么还不算“真正病例级” 你现在的 Agent 虽然也是对测试集里每个病例循环一遍，但它做决定时看的东西是： image_only 这个模型的整体 validation 分数 image_age_location 这个组合的整体 validation 分数 image_sex_location 这个组合的整体 validation 分数 这些分数对所有病例都一样。 比如在 lookahead 里，它脑子里想的是： image_age_location 整体是 0.6093 image_sex_location 整体是 0.6125 那它就会对所有病例都偏向同一条路线。 所以虽然代码是： 一个病例一个病例跑 但策略脑子里用的依据却是： 全局同一套分数表 这就会导致很多病例最后动作一样。

08 Validated Agent¶

This notebook rebuilds the agent baselines with a clean protocol:

  • policy_model_scores: use validation performance for agent decision making
  • report_model_scores: use test performance only for final reporting
In [1]:
import os
import json
import numpy as np
import pandas as pd
from pathlib import Path
from datetime import datetime
In [2]:
PROJECT_DIR = Path('/Users/applesues01/Documents/Medical_Agent')
DATA_DIR = PROJECT_DIR / 'data' / 'HAM10000'
SPLIT_DIR = DATA_DIR / 'splits'
SUPPORT_DIR = PROJECT_DIR / 'supports'

train_df = pd.read_csv(SPLIT_DIR / 'train.csv')
val_df = pd.read_csv(SPLIT_DIR / 'val.csv')
test_df = pd.read_csv(SPLIT_DIR / 'test.csv')

len(train_df), len(val_df), len(test_df)
Out[2]:
(7002, 1532, 1481)
In [3]:
# policy scores: validation only
policy_model_scores = {
    'image_only': {'macro_f1': 0.5907645547397793},
    'image_age': {'macro_f1': 0.5944472137149558},
    'image_sex': {'macro_f1': 0.6009349576323446},
    'image_location': {'macro_f1': 0.6037783548051701},
    'image_age_sex': {'macro_f1': 0.6040389979155417},
    'image_age_location': {'macro_f1': 0.6092976793614365},
    'image_sex_location': {'macro_f1': 0.6124672368332591},
    'image_all_metadata': {'macro_f1': 0.6039232782735721},
}

# report scores: test only
report_model_scores = {
    'image_only': {
        'accuracy': 0.7771775827143822,
        'balanced_accuracy': 0.6372819453123182,
        'macro_f1': 0.6083234281767498,
    },
    'image_age': {
        'accuracy': 0.7663740715732613,
        'balanced_accuracy': 0.5496342563971376,
        'macro_f1': 0.5573881698275872,
    },
    'image_sex': {
        'accuracy': 0.7866306549628629,
        'balanced_accuracy': 0.580348214532734,
        'macro_f1': 0.5790553454895507,
    },
    'image_location': {
        'accuracy': 0.7542201215395004,
        'balanced_accuracy': 0.5588534071029444,
        'macro_f1': 0.5648737073830044,
    },
    'image_age_sex': {
        'accuracy': 0.7933828494260635,
        'balanced_accuracy': 0.5822810012892775,
        'macro_f1': 0.5816659749556903,
    },
    'image_age_location': {
        'accuracy': 0.7832545577312626,
        'balanced_accuracy': 0.6028235514726813,
        'macro_f1': 0.6079100256641485,
    },
    'image_sex_location': {
        'accuracy': 0.7927076299797434,
        'balanced_accuracy': 0.5984582202347623,
        'macro_f1': 0.5964966418277308,
    },
    'image_all_metadata': {
        'accuracy': 0.799459824442944,
        'balanced_accuracy': 0.5940532294989452,
        'macro_f1': 0.6005789504507412,
    },
}

policy_model_scores, report_model_scores['image_only']
Out[3]:
({'image_only': {'macro_f1': 0.5907645547397793},
  'image_age': {'macro_f1': 0.5944472137149558},
  'image_sex': {'macro_f1': 0.6009349576323446},
  'image_location': {'macro_f1': 0.6037783548051701},
  'image_age_sex': {'macro_f1': 0.6040389979155417},
  'image_age_location': {'macro_f1': 0.6092976793614365},
  'image_sex_location': {'macro_f1': 0.6124672368332591},
  'image_all_metadata': {'macro_f1': 0.6039232782735721}},
 {'accuracy': 0.7771775827143822,
  'balanced_accuracy': 0.6372819453123182,
  'macro_f1': 0.6083234281767498})
In [4]:
def build_initial_state(row):
    return {
        'image_id': row['image_id'],
        'true_label': row['dx'],
        'known_metadata': {},
        'asked_questions': [],
        'done': False,
    }


def ask_question(state, row, question):
    assert question in ['age', 'sex', 'location']
    new_state = {
        'image_id': state['image_id'],
        'true_label': state['true_label'],
        'known_metadata': dict(state['known_metadata']),
        'asked_questions': list(state['asked_questions']),
        'done': state['done'],
    }

    if question == 'age':
        new_state['known_metadata']['age'] = row['age']
    elif question == 'sex':
        new_state['known_metadata']['sex'] = row['sex']
    elif question == 'location':
        new_state['known_metadata']['location'] = row['localization']

    new_state['asked_questions'].append(question)
    return new_state


def get_model_key_from_known_set(known_set):
    if known_set == set():
        return 'image_only'
    if known_set == {'age'}:
        return 'image_age'
    if known_set == {'sex'}:
        return 'image_sex'
    if known_set == {'location'}:
        return 'image_location'
    if known_set == {'age', 'sex'}:
        return 'image_age_sex'
    if known_set == {'age', 'location'}:
        return 'image_age_location'
    if known_set == {'sex', 'location'}:
        return 'image_sex_location'
    if known_set == {'age', 'sex', 'location'}:
        return 'image_all_metadata'
    raise ValueError(f'Unknown known_set: {known_set}')


def get_model_key_from_state(state):
    return get_model_key_from_known_set(set(state['known_metadata'].keys()))


def score_state(state):
    model_key = get_model_key_from_state(state)
    return policy_model_scores[model_key]['macro_f1']
In [5]:
FIXED_ORDER = ['age', 'sex', 'location']


def fixed_order_policy(state, max_questions=2):
    if len(state['asked_questions']) >= max_questions:
        return 'diagnose'
    for question in FIXED_ORDER:
        if question not in state['asked_questions']:
            return question
    return 'diagnose'


def uncertainty_triggered_policy(state, threshold, max_questions=2):
    current_score = score_state(state)
    if current_score >= threshold:
        return 'diagnose'
    return fixed_order_policy(state, max_questions=max_questions)


def lookahead_value_policy(state, max_questions=2):
    current_known = set(state['known_metadata'].keys())
    asked = set(state['asked_questions'])
    remaining_budget = max_questions - len(asked)

    if remaining_budget <= 0:
        return 'diagnose'

    candidate_questions = [q for q in ['age', 'sex', 'location'] if q not in asked]
    if not candidate_questions:
        return 'diagnose'

    current_score = policy_model_scores[get_model_key_from_known_set(current_known)]['macro_f1']
    best_question = None
    best_future_score = current_score

    for q1 in candidate_questions:
        known_after_q1 = current_known | {q1}
        best_path_score = policy_model_scores[get_model_key_from_known_set(known_after_q1)]['macro_f1']

        if remaining_budget >= 2:
            candidate_q2 = [q for q in ['age', 'sex', 'location'] if q not in known_after_q1]
            for q2 in candidate_q2:
                known_after_q2 = known_after_q1 | {q2}
                q2_score = policy_model_scores[get_model_key_from_known_set(known_after_q2)]['macro_f1']
                if q2_score > best_path_score:
                    best_path_score = q2_score

        if best_path_score > best_future_score:
            best_future_score = best_path_score
            best_question = q1

    if best_question is None:
        return 'diagnose'
    return best_question
In [6]:
def run_episode(row, policy_name, threshold=0.605, max_questions=2):
    state = build_initial_state(row)
    trajectory = []

    while not state['done']:
        if policy_name == 'fixed_order':
            action = fixed_order_policy(state, max_questions=max_questions)
        elif policy_name == 'uncertainty':
            action = uncertainty_triggered_policy(state, threshold=threshold, max_questions=max_questions)
        elif policy_name == 'lookahead':
            action = lookahead_value_policy(state, max_questions=max_questions)
        else:
            raise ValueError(policy_name)

        trajectory.append({
            'step': len(trajectory),
            'action': action,
            'model_key_before_action': get_model_key_from_state(state),
            'score_before_action': score_state(state),
            'known_metadata_before_action': dict(state['known_metadata']),
        })

        if action == 'diagnose':
            state['done'] = True
            state['final_action'] = 'diagnose'
            state['final_model_key'] = get_model_key_from_state(state)
            state['final_score'] = score_state(state)
            break

        state = ask_question(state, row, action)

    return trajectory, state


def evaluate_policy(policy_name, threshold=0.605, max_questions=2):
    records = []
    for idx in range(len(test_df)):
        row = test_df.iloc[idx]
        trajectory, final_state = run_episode(
            row,
            policy_name=policy_name,
            threshold=threshold,
            max_questions=max_questions,
        )
        records.append({
            'image_id': row['image_id'],
            'true_label': row['dx'],
            'asked_questions': list(final_state['asked_questions']),
            'num_questions': len(final_state['asked_questions']),
            'final_model_key': final_state['final_model_key'],
            'final_policy_score': final_state['final_score'],
            'known_metadata': dict(final_state['known_metadata']),
        })

    cases_df = pd.DataFrame(records)
    summary_df = cases_df.groupby('final_model_key').size().reset_index(name='count')
    summary_df['ratio'] = summary_df['count'] / len(cases_df)
    summary_df['test_accuracy'] = summary_df['final_model_key'].map(lambda k: report_model_scores[k]['accuracy'])
    summary_df['test_balanced_accuracy'] = summary_df['final_model_key'].map(lambda k: report_model_scores[k]['balanced_accuracy'])
    summary_df['test_macro_f1'] = summary_df['final_model_key'].map(lambda k: report_model_scores[k]['macro_f1'])

    result = {
        'agent_name': policy_name,
        'max_questions': max_questions,
        'avg_questions': float(cases_df['num_questions'].mean()),
        'expected_accuracy': float((summary_df['ratio'] * summary_df['test_accuracy']).sum()),
        'expected_balanced_accuracy': float((summary_df['ratio'] * summary_df['test_balanced_accuracy']).sum()),
        'expected_macro_f1': float((summary_df['ratio'] * summary_df['test_macro_f1']).sum()),
        'threshold': threshold if policy_name == 'uncertainty' else np.nan,
    }
    return cases_df, summary_df, result
In [7]:
fixed_cases, fixed_summary, fixed_result = evaluate_policy('fixed_order', max_questions=2)
unc_cases, unc_summary, unc_result = evaluate_policy('uncertainty', threshold=0.605, max_questions=2)
look_cases, look_summary, look_result = evaluate_policy('lookahead', max_questions=2)

pd.DataFrame([fixed_result, unc_result, look_result])
Out[7]:
agent_name max_questions avg_questions expected_accuracy expected_balanced_accuracy expected_macro_f1 threshold
0 fixed_order 2 2.0 0.793383 0.582281 0.581666 NaN
1 uncertainty 2 2.0 0.793383 0.582281 0.581666 0.605
2 lookahead 2 2.0 0.792708 0.598458 0.596497 NaN
In [8]:
save_dir = SUPPORT_DIR
date_tag = datetime.now().strftime('%Y-%m-%d')
time_tag = datetime.now().strftime('%H%M%S')

comparison_df = pd.DataFrame([fixed_result, unc_result, look_result])
comparison_path = save_dir / f'{date_tag}_{time_tag}_validated_agent_comparison.csv'
comparison_df.to_csv(comparison_path, index=False)

fixed_summary.to_csv(save_dir / f'{date_tag}_{time_tag}_validated_fixed_summary.csv', index=False)
unc_summary.to_csv(save_dir / f'{date_tag}_{time_tag}_validated_uncertainty_summary.csv', index=False)
look_summary.to_csv(save_dir / f'{date_tag}_{time_tag}_validated_lookahead_summary.csv', index=False)

print(comparison_path)
comparison_df

/Users/applesues01/Documents/Medical_Agent/supports/2026-08-03_185733_validated_agent_comparison.csv
Out[8]:
agent_name max_questions avg_questions expected_accuracy expected_balanced_accuracy expected_macro_f1 threshold
0 fixed_order 2 2.0 0.793383 0.582281 0.581666 NaN
1 uncertainty 2 2.0 0.793383 0.582281 0.581666 0.605
2 lookahead 2 2.0 0.792708 0.598458 0.596497 NaN