这个Agent就是针对单个病例具体问题具体分析了¶
In [8]:
# Cell 0
import os
import json
import numpy as np
import pandas as pd
from pathlib import Path
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader
from torchvision import transforms
from torchvision.models import resnet50
from PIL import Image
In [9]:
# Cell 1
PROJECT_DIR = Path("/Users/applesues01/Documents/Medical_Agent")
DATA_DIR = PROJECT_DIR / "data" / "HAM10000"
SPLIT_DIR = DATA_DIR / "splits"
CHECKPOINT_DIR = PROJECT_DIR / "checkpoints"
SUPPORT_DIR = PROJECT_DIR / "supports"
IMAGE_DIR1 = DATA_DIR / "HAM10000_images_part_1"
IMAGE_DIR2 = DATA_DIR / "HAM10000_images_part_2"
train_df = pd.read_csv(SPLIT_DIR / "train.csv")
val_df = pd.read_csv(SPLIT_DIR / "val.csv")
test_df = pd.read_csv(SPLIT_DIR / "test.csv")
len(train_df), len(val_df), len(test_df)
Out[9]:
(7002, 1532, 1481)
In [10]:
# Cell 2
LABEL_MAP = {
"akiec": 0,
"bcc": 1,
"bkl": 2,
"df": 3,
"mel": 4,
"nv": 5,
"vasc": 6,
}
IDX_TO_LABEL = {v: k for k, v in LABEL_MAP.items()}
CLASS_NAMES = [IDX_TO_LABEL[i] for i in range(7)]
CLASS_NAMES
Out[10]:
['akiec', 'bcc', 'bkl', 'df', 'mel', 'nv', 'vasc']
In [11]:
# Cell 3
device = torch.device("mps" if torch.backends.mps.is_available() else "cpu")
device
Out[11]:
device(type='mps')
In [12]:
# Cell 4
def resolve_image_path(image_id: str):
filename = f"{image_id}.jpg"
path1 = IMAGE_DIR1 / filename
path2 = IMAGE_DIR2 / filename
if path1.exists():
return path1
return path2
In [13]:
# Cell 5
eval_transform = transforms.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
])
In [14]:
# Cell 6
class HAMImageOnlyDataset(Dataset):
def __init__(self, dataframe, transform=None):
self.df = dataframe.reset_index(drop=True)
self.transform = transform
def __len__(self):
return len(self.df)
def __getitem__(self, idx):
row = self.df.iloc[idx]
image = Image.open(resolve_image_path(row["image_id"])).convert("RGB")
if self.transform:
image = self.transform(image)
label = LABEL_MAP[row["dx"]]
return {
"image_id": row["image_id"],
"image": image,
"label": label,
"true_label_name": row["dx"],
}
In [15]:
# Cell 7
test_dataset = HAMImageOnlyDataset(test_df, transform=eval_transform)
test_loader = DataLoader(
test_dataset,
batch_size=16,
shuffle=False,
num_workers=0
)
len(test_dataset)
Out[15]:
1481
In [16]:
# Cell 8
model = resnet50(weights=None)
model.fc = nn.Linear(model.fc.in_features, 7)
checkpoint_path = CHECKPOINT_DIR / "resnet50_image_only_finetuned_best.pth"
model.load_state_dict(torch.load(checkpoint_path, map_location=device))
model = model.to(device)
model.eval()
print("Loaded:", checkpoint_path)
Loaded: /Users/applesues01/Documents/Medical_Agent/checkpoints/resnet50_image_only_finetuned_best.pth
In [17]:
# Cell 9
@torch.no_grad()
def collect_image_only_predictions(model, data_loader):
all_rows = []
for batch in data_loader:
images = batch["image"].to(device)
logits = model(images)
probs = torch.softmax(logits, dim=1)
pred_idx = probs.argmax(dim=1)
max_prob = probs.max(dim=1).values
for i in range(len(pred_idx)):
prob_vector = probs[i].cpu().numpy()
row = {
"image_id": batch["image_id"][i],
"true_label": batch["true_label_name"][i],
"pred_label": IDX_TO_LABEL[int(pred_idx[i].cpu().item())],
"pred_idx": int(pred_idx[i].cpu().item()),
"max_prob": float(max_prob[i].cpu().item()),
}
for cls_idx, cls_name in enumerate(CLASS_NAMES):
row[f"prob_{cls_name}"] = float(prob_vector[cls_idx])
all_rows.append(row)
return pd.DataFrame(all_rows)
In [18]:
# Cell 10
image_only_case_df = collect_image_only_predictions(model, test_loader)
image_only_case_df.head()
Out[18]:
| image_id | true_label | pred_label | pred_idx | max_prob | prob_akiec | prob_bcc | prob_bkl | prob_df | prob_mel | prob_nv | prob_vasc | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | ISIC_0025837 | bkl | bkl | 2 | 0.972961 | 0.004813 | 0.001014 | 0.972961 | 0.000886 | 0.018995 | 0.001147 | 0.000185 |
| 1 | ISIC_0025209 | bkl | bkl | 2 | 0.407122 | 0.220992 | 0.070574 | 0.407122 | 0.207246 | 0.073726 | 0.019543 | 0.000797 |
| 2 | ISIC_0029161 | bkl | bkl | 2 | 0.793358 | 0.000883 | 0.001113 | 0.793358 | 0.000149 | 0.074286 | 0.127832 | 0.002381 |
| 3 | ISIC_0026273 | bkl | bkl | 2 | 0.802558 | 0.000078 | 0.005748 | 0.802558 | 0.000180 | 0.094588 | 0.055271 | 0.041577 |
| 4 | ISIC_0025819 | bkl | bkl | 2 | 0.978878 | 0.000174 | 0.000190 | 0.978878 | 0.001796 | 0.009443 | 0.009492 | 0.000025 |
In [19]:
# Cell 11
image_only_case_df["max_prob"].describe()
Out[19]:
count 1481.000000 mean 0.828062 std 0.188904 min 0.240508 25% 0.699200 50% 0.910106 75% 0.989811 max 0.999999 Name: max_prob, dtype: float64
In [20]:
# Cell 12
image_only_case_df.sort_values("max_prob").head(10)
Out[20]:
| image_id | true_label | pred_label | pred_idx | max_prob | prob_akiec | prob_bcc | prob_bkl | prob_df | prob_mel | prob_nv | prob_vasc | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 1103 | ISIC_0028214 | nv | akiec | 0 | 0.240508 | 0.240508 | 0.239235 | 0.238439 | 0.001353 | 0.004728 | 0.201975 | 7.376333e-02 |
| 193 | ISIC_0030080 | mel | bcc | 1 | 0.299620 | 0.185799 | 0.299620 | 0.111202 | 0.004639 | 0.138884 | 0.259430 | 4.254647e-04 |
| 36 | ISIC_0032898 | bkl | nv | 5 | 0.302573 | 0.000293 | 0.050452 | 0.275924 | 0.003778 | 0.176799 | 0.302573 | 1.901817e-01 |
| 1239 | ISIC_0029938 | nv | mel | 4 | 0.307989 | 0.009250 | 0.026222 | 0.208872 | 0.145729 | 0.307989 | 0.300720 | 1.218622e-03 |
| 1240 | ISIC_0024777 | nv | mel | 4 | 0.307989 | 0.009250 | 0.026222 | 0.208872 | 0.145729 | 0.307989 | 0.300720 | 1.218622e-03 |
| 137 | ISIC_0029427 | bkl | akiec | 0 | 0.314603 | 0.314603 | 0.007415 | 0.312598 | 0.136129 | 0.099714 | 0.128612 | 9.288199e-04 |
| 1322 | ISIC_0031192 | nv | bkl | 2 | 0.318004 | 0.248337 | 0.004944 | 0.318004 | 0.001515 | 0.249158 | 0.178041 | 1.320831e-06 |
| 351 | ISIC_0024739 | mel | akiec | 0 | 0.323613 | 0.323613 | 0.004626 | 0.308846 | 0.000022 | 0.265395 | 0.097498 | 1.654940e-08 |
| 61 | ISIC_0029770 | bkl | mel | 4 | 0.324856 | 0.155010 | 0.000347 | 0.195253 | 0.000138 | 0.324856 | 0.324391 | 3.719530e-06 |
| 205 | ISIC_0027352 | mel | bkl | 2 | 0.333486 | 0.198477 | 0.007150 | 0.333486 | 0.225864 | 0.214819 | 0.020193 | 1.090698e-05 |
In [21]:
# Cell 13
image_only_case_df.sort_values("max_prob", ascending=False).head(10)
Out[21]:
| image_id | true_label | pred_label | pred_idx | max_prob | prob_akiec | prob_bcc | prob_bkl | prob_df | prob_mel | prob_nv | prob_vasc | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 371 | ISIC_0024706 | vasc | vasc | 6 | 0.999999 | 5.348534e-13 | 4.237151e-10 | 3.467123e-11 | 1.482642e-12 | 3.971183e-10 | 0.000001 | 9.999988e-01 |
| 368 | ISIC_0031065 | vasc | vasc | 6 | 0.999996 | 3.378164e-11 | 1.356773e-06 | 8.096805e-09 | 2.017714e-11 | 6.565480e-08 | 0.000003 | 9.999957e-01 |
| 364 | ISIC_0029608 | vasc | vasc | 6 | 0.999995 | 1.714149e-10 | 3.998913e-08 | 5.497389e-08 | 1.270005e-09 | 1.604129e-06 | 0.000004 | 9.999945e-01 |
| 785 | ISIC_0027001 | nv | nv | 5 | 0.999989 | 8.091762e-09 | 7.306994e-09 | 5.145294e-06 | 4.169624e-08 | 5.891593e-06 | 0.999989 | 1.304085e-07 |
| 972 | ISIC_0027223 | nv | nv | 5 | 0.999985 | 3.405929e-07 | 5.237956e-08 | 1.736144e-07 | 1.083372e-08 | 1.459452e-05 | 0.999985 | 5.922883e-09 |
| 597 | ISIC_0028318 | nv | nv | 5 | 0.999974 | 2.549434e-09 | 7.658581e-08 | 2.022613e-05 | 4.551421e-07 | 4.472800e-06 | 0.999974 | 2.086867e-07 |
| 634 | ISIC_0026300 | nv | nv | 5 | 0.999964 | 7.188337e-08 | 6.294847e-08 | 9.683251e-07 | 8.649195e-06 | 2.593950e-05 | 0.999964 | 2.351306e-08 |
| 588 | ISIC_0026628 | nv | nv | 5 | 0.999964 | 3.234657e-07 | 1.087137e-07 | 4.385951e-06 | 2.248856e-06 | 2.925699e-05 | 0.999964 | 1.207314e-08 |
| 850 | ISIC_0030685 | nv | nv | 5 | 0.999943 | 6.310223e-08 | 5.300025e-08 | 3.129247e-06 | 1.732093e-05 | 3.596670e-05 | 0.999943 | 1.338907e-07 |
| 596 | ISIC_0031508 | nv | nv | 5 | 0.999938 | 1.237050e-08 | 9.408037e-09 | 6.599078e-07 | 1.952678e-08 | 6.101678e-05 | 0.999938 | 3.789528e-09 |
In [22]:
# Cell 14
save_path = SUPPORT_DIR / "image_only_case_level_predictions.csv"
image_only_case_df.to_csv(save_path, index=False)
print(save_path)
/Users/applesues01/Documents/Medical_Agent/supports/image_only_case_level_predictions.csv
这样子看，有些看图就行，有些看不出来的，也许问问题会有帮助？¶
In [23]:
# Cell 15
BEST_QUESTION_ORDER = ["age", "location", "sex"]
BEST_QUESTION_ORDER
Out[23]:
['age', 'location', 'sex']
In [24]:
# Cell 16
def build_case_level_state(row):
return {
"image_id": row["image_id"],
"true_label": row["true_label"],
"pred_label": row["pred_label"],
"max_prob": row["max_prob"],
"known_metadata": {},
"asked_questions": [],
"done": False,
}
In [25]:
# Cell 17
def ask_case_question(state, meta_row, question):
new_state = {
"image_id": state["image_id"],
"true_label": state["true_label"],
"pred_label": state["pred_label"],
"max_prob": state["max_prob"],
"known_metadata": dict(state["known_metadata"]),
"asked_questions": list(state["asked_questions"]),
"done": state["done"],
}
if question == "age":
new_state["known_metadata"]["age"] = meta_row["age"]
elif question == "location":
new_state["known_metadata"]["location"] = meta_row["localization"]
elif question == "sex":
new_state["known_metadata"]["sex"] = meta_row["sex"]
new_state["asked_questions"].append(question)
return new_state
In [26]:
# Cell 18
metadata_lookup_df = test_df[["image_id", "dx", "age", "sex", "localization"]].copy()
metadata_lookup_df.head()
Out[26]:
| image_id | dx | age | sex | localization | |
|---|---|---|---|---|---|
| 0 | ISIC_0025837 | bkl | 70.0 | female | back |
| 1 | ISIC_0025209 | bkl | 70.0 | female | back |
| 2 | ISIC_0029161 | bkl | 60.0 | male | chest |
| 3 | ISIC_0026273 | bkl | 60.0 | male | chest |
| 4 | ISIC_0025819 | bkl | 75.0 | female | face |
In [27]:
# Cell 19
metadata_lookup = {
row["image_id"]: row
for _, row in metadata_lookup_df.iterrows()
}
len(metadata_lookup)
Out[27]:
1481
In [28]:
# Cell 20
def case_level_uncertainty_policy(state, threshold=0.80, max_questions=2):
if state["max_prob"] >= threshold:
return "diagnose"
if len(state["asked_questions"]) >= max_questions:
return "diagnose"
for q in BEST_QUESTION_ORDER:
if q not in state["asked_questions"]:
return q
return "diagnose"
In [29]:
# Cell 20
def case_level_uncertainty_policy(state, threshold=0.80, max_questions=2):
if state["max_prob"] >= threshold:
return "diagnose"
if len(state["asked_questions"]) >= max_questions:
return "diagnose"
for q in BEST_QUESTION_ORDER:
if q not in state["asked_questions"]:
return q
return "diagnose"
In [30]:
# Cell 21
def run_case_level_episode(case_row, threshold=0.80, max_questions=2):
state = build_case_level_state(case_row)
meta_row = metadata_lookup[state["image_id"]]
trajectory = []
while not state["done"]:
action = case_level_uncertainty_policy(
state,
threshold=threshold,
max_questions=max_questions
)
trajectory.append({
"step": len(trajectory),
"action": action,
"max_prob": state["max_prob"],
"known_metadata_before_action": dict(state["known_metadata"]),
})
if action == "diagnose":
state["done"] = True
state["final_action"] = "diagnose"
break
state = ask_case_question(state, meta_row, action)
return trajectory, state
In [31]:
# Cell 22
trajectory, final_state = run_case_level_episode(
image_only_case_df.iloc[0],
threshold=0.80,
max_questions=2
)
for item in trajectory:
print(item)
print("\nfinal_state:")
final_state
{'step': 0, 'action': 'diagnose', 'max_prob': np.float64(0.9729606509208679), 'known_metadata_before_action': {}}
final_state:
Out[31]:
{'image_id': 'ISIC_0025837',
'true_label': 'bkl',
'pred_label': 'bkl',
'max_prob': np.float64(0.9729606509208679),
'known_metadata': {},
'asked_questions': [],
'done': True,
'final_action': 'diagnose'}
In [32]:
image_only_case_df.sort_values("max_prob", ascending=False).head(1)
image_only_case_df.sort_values("max_prob", ascending=True).head(1)
Out[32]:
| image_id | true_label | pred_label | pred_idx | max_prob | prob_akiec | prob_bcc | prob_bkl | prob_df | prob_mel | prob_nv | prob_vasc | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 1103 | ISIC_0028214 | nv | akiec | 0 | 0.240508 | 0.240508 | 0.239235 | 0.238439 | 0.001353 | 0.004728 | 0.201975 | 0.073763 |
In [33]:
# Cell 23: 低置信病例测试
low_case = image_only_case_df.sort_values("max_prob", ascending=True).iloc[0]
trajectory, final_state = run_case_level_episode(
low_case,
threshold=0.80,
max_questions=2
)
print("low_case:")
print(low_case[["image_id", "true_label", "pred_label", "max_prob"]])
print("\ntrajectory:")
for item in trajectory:
print(item)
print("\nfinal_state:")
final_state
low_case:
image_id ISIC_0028214
true_label nv
pred_label akiec
max_prob 0.240508
Name: 1103, dtype: object
trajectory:
{'step': 0, 'action': 'age', 'max_prob': np.float64(0.24050791561603546), 'known_metadata_before_action': {}}
{'step': 1, 'action': 'location', 'max_prob': np.float64(0.24050791561603546), 'known_metadata_before_action': {'age': 50.0}}
{'step': 2, 'action': 'diagnose', 'max_prob': np.float64(0.24050791561603546), 'known_metadata_before_action': {'age': 50.0, 'location': 'scalp'}}
final_state:
Out[33]:
{'image_id': 'ISIC_0028214',
'true_label': 'nv',
'pred_label': 'akiec',
'max_prob': np.float64(0.24050791561603546),
'known_metadata': {'age': 50.0, 'location': 'scalp'},
'asked_questions': ['age', 'location'],
'done': True,
'final_action': 'diagnose'}
概率没更新是因为他只是觉得概率低就提问，但并没有把提问结果送回模型重新推理，但是至少证明了模型还是会提问的，能区分哪些需要问，哪些不需要问¶
In [52]:
# Cell 24
CLASS_NAMES = ["akiec", "bcc", "bkl", "df", "mel", "nv", "vasc"]
COMBINATION_MODEL_CONFIG = {
frozenset(): {
"name": "image_only",
"type": "image_only",
"checkpoint": "/Users/applesues01/Documents/Medical_Agent/checkpoints/resnet50_image_only_finetuned_best.pth",
"metadata_features": [],
"metadata_embed_dim": None,
},
frozenset({"age"}): {
"name": "image_age",
"type": "fusion",
"checkpoint": "/Users/applesues01/Documents/Medical_Agent/checkpoints/image_age_best.pth",
"metadata_features": ["age"],
"metadata_embed_dim": 256,
},
frozenset({"sex"}): {
"name": "image_sex",
"type": "fusion",
"checkpoint": "/Users/applesues01/Documents/Medical_Agent/checkpoints/image_sex_best.pth",
"metadata_features": ["sex"],
"metadata_embed_dim": 32,
},
frozenset({"location"}): {
"name": "image_location",
"type": "fusion",
"checkpoint": "/Users/applesues01/Documents/Medical_Agent/checkpoints/image_location_best.pth",
"metadata_features": ["location"],
"metadata_embed_dim": 64,
},
frozenset({"age", "sex"}): {
"name": "image_age_sex",
"type": "fusion",
"checkpoint": "/Users/applesues01/Documents/Medical_Agent/checkpoints/image_age_sex_best.pth",
"metadata_features": ["age", "sex"],
"metadata_embed_dim": 32,
},
frozenset({"age", "location"}): {
"name": "image_age_location",
"type": "fusion",
"checkpoint": "/Users/applesues01/Documents/Medical_Agent/checkpoints/image_age_location_best.pth",
"metadata_features": ["age", "location"],
"metadata_embed_dim": None, # raw
},
frozenset({"sex", "location"}): {
"name": "image_sex_location",
"type": "fusion",
"checkpoint": "/Users/applesues01/Documents/Medical_Agent/checkpoints/image_sex_location_best.pth",
"metadata_features": ["sex", "location"],
"metadata_embed_dim": 128,
},
frozenset({"age", "sex", "location"}): {
"name": "image_all_metadata",
"type": "fusion",
"checkpoint": "/Users/applesues01/Documents/Medical_Agent/checkpoints/image_all_metadata_best.pth",
"metadata_features": ["age", "sex", "location"],
"metadata_embed_dim": 32,
},
}
COMBINATION_MODEL_CONFIG[frozenset({"age", "location"})]
Out[52]:
{'name': 'image_age_location',
'type': 'fusion',
'checkpoint': '/Users/applesues01/Documents/Medical_Agent/checkpoints/image_age_location_best.pth',
'metadata_features': ['age', 'location'],
'metadata_embed_dim': None}
In [53]:
# Cell 25
train_age_mean = train_df["age"].mean()
LOCATIONS = [
"scalp", "ear", "face", "back", "trunk", "chest",
"upper extremity", "abdomen", "unknown", "lower extremity",
"genital", "neck", "hand", "foot", "acral"
]
SEX_MAP = {
"male": [1.0, 0.0, 0.0],
"female": [0.0, 1.0, 0.0],
"unknown": [0.0, 0.0, 1.0],
}
def build_metadata_vector_from_row(meta_row, selected_features):
feats = []
if "age" in selected_features:
age = meta_row["age"]
if pd.isna(age):
age = train_age_mean
feats.append(float(age) / 100.0)
if "sex" in selected_features:
sex_key = meta_row["sex"] if meta_row["sex"] in SEX_MAP else "unknown"
feats.extend(SEX_MAP[sex_key])
if "location" in selected_features:
loc_key = meta_row["localization"] if meta_row["localization"] in LOCATIONS else "unknown"
loc_vector = [0.0] * len(LOCATIONS)
loc_vector[LOCATIONS.index(loc_key)] = 1.0
feats.extend(loc_vector)
return torch.tensor(feats, dtype=torch.float32).unsqueeze(0)
In [54]:
# Cell 26
class RawFusionClassifier(nn.Module):
def __init__(self, metadata_input_dim, num_classes=7):
super().__init__()
self.classifier = nn.Sequential(
nn.Linear(2048 + metadata_input_dim, 512),
nn.ReLU(),
nn.Dropout(0.3),
nn.Linear(512, 128),
nn.ReLU(),
nn.Linear(128, num_classes),
)
def forward(self, image_features, metadata):
fused = torch.cat([image_features, metadata], dim=1)
return self.classifier(fused)
class MetadataEncoderFusionClassifier(nn.Module):
def __init__(self, metadata_input_dim, metadata_embed_dim=64, num_classes=7):
super().__init__()
self.metadata_encoder = nn.Sequential(
nn.Linear(metadata_input_dim, metadata_embed_dim),
nn.BatchNorm1d(metadata_embed_dim),
nn.ReLU(),
nn.Dropout(0.2),
)
self.classifier = nn.Sequential(
nn.Linear(2048 + metadata_embed_dim, 512),
nn.ReLU(),
nn.Dropout(0.3),
nn.Linear(512, 128),
nn.ReLU(),
nn.Linear(128, num_classes),
)
def forward(self, image_features, metadata):
metadata_features = self.metadata_encoder(metadata)
fused = torch.cat([image_features, metadata_features], dim=1)
return self.classifier(fused)
In [55]:
# Cell 27
class ResNet50FeatureExtractor(nn.Module):
def __init__(self, backbone):
super().__init__()
self.features = nn.Sequential(*list(backbone.children())[:-1])
def forward(self, x):
x = self.features(x)
return torch.flatten(x, 1)
In [56]:
# Cell 28
image_only_backbone = resnet50(weights=None)
image_only_backbone.fc = nn.Linear(image_only_backbone.fc.in_features, 7)
image_only_backbone.load_state_dict(
torch.load(
"/Users/applesues01/Documents/Medical_Agent/checkpoints/resnet50_image_only_finetuned_best.pth",
map_location=device
)
)
image_only_backbone = image_only_backbone.to(device)
image_only_backbone.eval()
feature_extractor = ResNet50FeatureExtractor(image_only_backbone).to(device)
feature_extractor.eval()
for p in feature_extractor.parameters():
p.requires_grad = False
print("feature extractor ready")
feature extractor ready
In [57]:
# Cell 29
loaded_models = {}
def load_combination_model(known_keys):
key = frozenset(known_keys)
config = COMBINATION_MODEL_CONFIG[key]
if key in loaded_models:
return loaded_models[key], config
if config["type"] == "image_only":
model = resnet50(weights=None)
model.fc = nn.Linear(model.fc.in_features, 7)
model.load_state_dict(torch.load(config["checkpoint"], map_location=device))
model = model.to(device)
model.eval()
loaded_models[key] = model
return model, config
dummy_meta = build_metadata_vector_from_row(
{
"age": train_age_mean,
"sex": "unknown",
"localization": "unknown"
},
config["metadata_features"]
)
metadata_dim = len(dummy_meta.squeeze(0))
if config["metadata_embed_dim"] is None:
model = RawFusionClassifier(
metadata_input_dim=metadata_dim,
num_classes=7
)
else:
model = MetadataEncoderFusionClassifier(
metadata_input_dim=metadata_dim,
metadata_embed_dim=config["metadata_embed_dim"],
num_classes=7
)
model.load_state_dict(torch.load(config["checkpoint"], map_location=device))
model = model.to(device)
model.eval()
loaded_models[key] = model
return model, config
In [58]:
# Cell 30
@torch.no_grad()
def predict_with_known_metadata(image_tensor, meta_row, known_keys):
model, config = load_combination_model(known_keys)
image_tensor = image_tensor.unsqueeze(0).to(device)
if config["type"] == "image_only":
logits = model(image_tensor)
else:
image_features = feature_extractor(image_tensor)
metadata_tensor = build_metadata_vector_from_row(
meta_row,
config["metadata_features"]
).to(device)
logits = model(image_features, metadata_tensor)
probs = torch.softmax(logits, dim=1).squeeze(0).cpu().numpy()
pred_idx = int(np.argmax(probs))
max_prob = float(np.max(probs))
return {
"model_name": config["name"],
"pred_label": CLASS_NAMES[pred_idx],
"pred_idx": pred_idx,
"max_prob": max_prob,
"prob_vector": probs,
}
In [59]:
# Cell 31: 从 image_id 找到原始 metadata
metadata_lookup = {
row["image_id"]: row
for _, row in metadata_lookup_df.iterrows()
}
len(metadata_lookup)
# Cell 32: 取一个低置信病例，做逐步重新推理
low_case = image_only_case_df.sort_values("max_prob", ascending=True).iloc[0]
low_case
# Cell 33: 找到这张图和它的metadata
image_id = low_case["image_id"]
meta_row = metadata_lookup[image_id]
raw_image = Image.open(resolve_image_path(image_id)).convert("RGB")
image_tensor = eval_transform(raw_image)
print("image_id:", image_id)
print("true_label:", meta_row["dx"])
print("age:", meta_row["age"])
print("sex:", meta_row["sex"])
print("location:", meta_row["localization"])
# Cell 34: 在不同已知信息状态下重新预测
pred_image_only = predict_with_known_metadata(
image_tensor=image_tensor,
meta_row=meta_row,
known_keys=[]
)
pred_age = predict_with_known_metadata(
image_tensor=image_tensor,
meta_row=meta_row,
known_keys=["age"]
)
pred_age_location = predict_with_known_metadata(
image_tensor=image_tensor,
meta_row=meta_row,
known_keys=["age", "location"]
)
pred_sex_location = predict_with_known_metadata(
image_tensor=image_tensor,
meta_row=meta_row,
known_keys=["sex", "location"]
)
pred_all = predict_with_known_metadata(
image_tensor=image_tensor,
meta_row=meta_row,
known_keys=["age", "sex", "location"]
)
pred_image_only, pred_age, pred_age_location
image_id: ISIC_0028214 true_label: nv age: 50.0 sex: male location: scalp
Out[59]:
({'model_name': 'image_only',
'pred_label': 'akiec',
'pred_idx': 0,
'max_prob': 0.24050791561603546,
'prob_vector': array([0.24050792, 0.23923509, 0.23843858, 0.00135264, 0.00472786,
0.20197459, 0.07376333], dtype=float32)},
{'model_name': 'image_age',
'pred_label': 'bkl',
'pred_idx': 2,
'max_prob': 0.8971191644668579,
'prob_vector': array([3.0671773e-04, 4.2722044e-03, 8.9711916e-01, 2.4032412e-04,
2.4110173e-05, 8.6835712e-02, 1.1201879e-02], dtype=float32)},
{'model_name': 'image_age_location',
'pred_label': 'bkl',
'pred_idx': 2,
'max_prob': 0.7396444082260132,
'prob_vector': array([1.6577450e-03, 6.2123658e-03, 7.3964441e-01, 9.5286116e-04,
4.1727739e-04, 2.1935184e-01, 3.1763509e-02], dtype=float32)})
注意看cell编号，下面开始跑35¶
In [60]:
# Cell 35: 整理成可读表格
case_compare_df = pd.DataFrame([
{
"state": "image_only",
"model_name": pred_image_only["model_name"],
"pred_label": pred_image_only["pred_label"],
"max_prob": pred_image_only["max_prob"],
},
{
"state": "age",
"model_name": pred_age["model_name"],
"pred_label": pred_age["pred_label"],
"max_prob": pred_age["max_prob"],
},
{
"state": "age_location",
"model_name": pred_age_location["model_name"],
"pred_label": pred_age_location["pred_label"],
"max_prob": pred_age_location["max_prob"],
},
{
"state": "sex_location",
"model_name": pred_sex_location["model_name"],
"pred_label": pred_sex_location["pred_label"],
"max_prob": pred_sex_location["max_prob"],
},
{
"state": "all_metadata",
"model_name": pred_all["model_name"],
"pred_label": pred_all["pred_label"],
"max_prob": pred_all["max_prob"],
},
])
case_compare_df
Out[60]:
| state | model_name | pred_label | max_prob | |
|---|---|---|---|---|
| 0 | image_only | image_only | akiec | 0.240508 |
| 1 | age | image_age | bkl | 0.897119 |
| 2 | age_location | image_age_location | bkl | 0.739644 |
| 3 | sex_location | image_sex_location | bkl | 0.829821 |
| 4 | all_metadata | image_all_metadata | nv | 0.666372 |
In [61]:
# Cell 36: 看完整7类概率变化
prob_compare_df = pd.DataFrame({
"class": CLASS_NAMES,
"image_only": pred_image_only["prob_vector"],
"age": pred_age["prob_vector"],
"age_location": pred_age_location["prob_vector"],
"sex_location": pred_sex_location["prob_vector"],
"all_metadata": pred_all["prob_vector"],
})
prob_compare_df
Out[61]:
| class | image_only | age | age_location | sex_location | all_metadata | |
|---|---|---|---|---|---|---|
| 0 | akiec | 0.240508 | 0.000307 | 0.001658 | 0.000087 | 0.000054 |
| 1 | bcc | 0.239235 | 0.004272 | 0.006212 | 0.005142 | 0.000885 |
| 2 | bkl | 0.238439 | 0.897119 | 0.739644 | 0.829821 | 0.313802 |
| 3 | df | 0.001353 | 0.000240 | 0.000953 | 0.000975 | 0.000380 |
| 4 | mel | 0.004728 | 0.000024 | 0.000417 | 0.000113 | 0.000005 |
| 5 | nv | 0.201975 | 0.086836 | 0.219352 | 0.140592 | 0.666372 |
| 6 | vasc | 0.073763 | 0.011202 | 0.031764 | 0.023271 | 0.018502 |
证明了确实提问不同概率也会不同！！！¶
In [62]:
# Cell 37
QUESTION_CANDIDATES = ["age", "sex", "location"]
QUESTION_BUDGET = 2
CONFIDENCE_THRESHOLD = 0.80
In [63]:
# Cell 38
def build_dynamic_state(case_row):
image_id = case_row["image_id"]
meta_row = metadata_lookup[image_id]
raw_image = Image.open(resolve_image_path(image_id)).convert("RGB")
image_tensor = eval_transform(raw_image)
initial_pred = predict_with_known_metadata(
image_tensor=image_tensor,
meta_row=meta_row,
known_keys=[]
)
return {
"image_id": image_id,
"true_label": case_row["true_label"],
"image_tensor": image_tensor,
"meta_row": meta_row,
"known_keys": [],
"asked_questions": [],
"current_pred": initial_pred,
"done": False,
}
In [64]:
# Cell 39
def choose_best_next_question(state):
best_question = None
best_pred = None
best_score = state["current_pred"]["max_prob"]
for q in QUESTION_CANDIDATES:
if q in state["asked_questions"]:
continue
next_keys = list(state["known_keys"]) + [q]
pred = predict_with_known_metadata(
image_tensor=state["image_tensor"],
meta_row=state["meta_row"],
known_keys=next_keys
)
if pred["max_prob"] > best_score:
best_score = pred["max_prob"]
best_question = q
best_pred = pred
return best_question, best_pred
In [65]:
# Cell 40
def run_dynamic_case_episode(case_row, threshold=0.80, max_questions=2):
state = build_dynamic_state(case_row)
trajectory = []
while not state["done"]:
current_pred = state["current_pred"]
trajectory.append({
"step": len(trajectory),
"known_keys": list(state["known_keys"]),
"asked_questions": list(state["asked_questions"]),
"pred_label": current_pred["pred_label"],
"max_prob": current_pred["max_prob"],
})
if current_pred["max_prob"] >= threshold:
state["done"] = True
state["final_action"] = "diagnose"
break
if len(state["asked_questions"]) >= max_questions:
state["done"] = True
state["final_action"] = "diagnose"
break
best_question, best_pred = choose_best_next_question(state)
if best_question is None:
state["done"] = True
state["final_action"] = "diagnose"
break
state["asked_questions"].append(best_question)
state["known_keys"].append(best_question)
state["current_pred"] = best_pred
return trajectory, state
In [66]:
# Cell 41
trajectory, final_state = run_dynamic_case_episode(
low_case,
threshold=0.80,
max_questions=2
)
for item in trajectory:
print(item)
print("\nfinal_state:")
print({
"image_id": final_state["image_id"],
"true_label": final_state["true_label"],
"asked_questions": final_state["asked_questions"],
"final_pred_label": final_state["current_pred"]["pred_label"],
"final_max_prob": final_state["current_pred"]["max_prob"],
})
{'step': 0, 'known_keys': [], 'asked_questions': [], 'pred_label': 'akiec', 'max_prob': 0.24050791561603546}
{'step': 1, 'known_keys': ['age'], 'asked_questions': ['age'], 'pred_label': 'bkl', 'max_prob': 0.8971191644668579}
final_state:
{'image_id': 'ISIC_0028214', 'true_label': 'nv', 'asked_questions': ['age'], 'final_pred_label': 'bkl', 'final_max_prob': 0.8971191644668579}
In [67]:
# Cell 42: 跑完整个测试集
all_dynamic_records = []
for idx in range(len(image_only_case_df)):
case_row = image_only_case_df.iloc[idx]
trajectory, final_state = run_dynamic_case_episode(
case_row,
threshold=0.80,
max_questions=2
)
initial_correct = (case_row["pred_label"] == case_row["true_label"])
final_correct = (final_state["current_pred"]["pred_label"] == final_state["true_label"])
all_dynamic_records.append({
"image_id": final_state["image_id"],
"true_label": final_state["true_label"],
"initial_pred_label": case_row["pred_label"],
"initial_max_prob": case_row["max_prob"],
"initial_correct": initial_correct,
"final_pred_label": final_state["current_pred"]["pred_label"],
"final_max_prob": final_state["current_pred"]["max_prob"],
"final_correct": final_correct,
"asked_questions": list(final_state["asked_questions"]),
"num_questions": len(final_state["asked_questions"]),
"known_keys": list(final_state["known_keys"]),
})
dynamic_agent_df = pd.DataFrame(all_dynamic_records)
dynamic_agent_df.head()
Out[67]:
| image_id | true_label | initial_pred_label | initial_max_prob | initial_correct | final_pred_label | final_max_prob | final_correct | asked_questions | num_questions | known_keys | |
|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | ISIC_0025837 | bkl | bkl | 0.972961 | True | bkl | 0.972961 | True | [] | 0 | [] |
| 1 | ISIC_0025209 | bkl | bkl | 0.407122 | True | bkl | 0.874424 | True | [age, sex] | 2 | [age, sex] |
| 2 | ISIC_0029161 | bkl | bkl | 0.793358 | True | bkl | 0.987926 | True | [sex] | 1 | [sex] |
| 3 | ISIC_0026273 | bkl | bkl | 0.802558 | True | bkl | 0.802558 | True | [] | 0 | [] |
| 4 | ISIC_0025819 | bkl | bkl | 0.978878 | True | bkl | 0.978878 | True | [] | 0 | [] |
In [68]:
# Cell 43: 基本统计
print("Total cases:", len(dynamic_agent_df))
print("Average questions:", dynamic_agent_df["num_questions"].mean())
print("\nQuestion count distribution:")
print(dynamic_agent_df["num_questions"].value_counts().sort_index())
print("\nInitial accuracy:")
print(dynamic_agent_df["initial_correct"].mean())
print("\nFinal accuracy:")
print(dynamic_agent_df["final_correct"].mean())
Total cases: 1481 Average questions: 0.36934503713706957 Question count distribution: num_questions 0 973 1 469 2 39 Name: count, dtype: int64 Initial accuracy: 0.7771775827143822 Final accuracy: 0.7825793382849426
In [69]:
# Cell 44: 初始 vs 最终，哪些变好了，哪些变坏了
dynamic_agent_df["changed_prediction"] = (
dynamic_agent_df["initial_pred_label"] != dynamic_agent_df["final_pred_label"]
)
dynamic_agent_df["improved"] = (
(dynamic_agent_df["initial_correct"] == False) &
(dynamic_agent_df["final_correct"] == True)
)
dynamic_agent_df["worsened"] = (
(dynamic_agent_df["initial_correct"] == True) &
(dynamic_agent_df["final_correct"] == False)
)
dynamic_agent_df["still_wrong"] = (
(dynamic_agent_df["initial_correct"] == False) &
(dynamic_agent_df["final_correct"] == False)
)
dynamic_agent_df["still_correct"] = (
(dynamic_agent_df["initial_correct"] == True) &
(dynamic_agent_df["final_correct"] == True)
)
dynamic_agent_df[
["changed_prediction", "improved", "worsened", "still_wrong", "still_correct"]
].mean()
Out[69]:
changed_prediction 0.104659 improved 0.045240 worsened 0.039838 still_wrong 0.177583 still_correct 0.737340 dtype: float64
In [70]:
# Cell 45: 高置信错误定义
HIGH_CONF_THRESHOLD = 0.80
dynamic_agent_df["initial_high_conf_wrong"] = (
(dynamic_agent_df["initial_max_prob"] >= HIGH_CONF_THRESHOLD) &
(dynamic_agent_df["initial_correct"] == False)
)
dynamic_agent_df["final_high_conf_wrong"] = (
(dynamic_agent_df["final_max_prob"] >= HIGH_CONF_THRESHOLD) &
(dynamic_agent_df["final_correct"] == False)
)
dynamic_agent_df["unsafe_confidence_increase"] = (
(dynamic_agent_df["initial_correct"] == False) &
(dynamic_agent_df["final_correct"] == False) &
(dynamic_agent_df["final_max_prob"] > dynamic_agent_df["initial_max_prob"])
)
dynamic_agent_df[
["initial_high_conf_wrong", "final_high_conf_wrong", "unsafe_confidence_increase"]
].mean()
Out[70]:
initial_high_conf_wrong 0.057394 final_high_conf_wrong 0.181634 unsafe_confidence_increase 0.116138 dtype: float64
In [71]:
# Cell 46: 看最危险的病例
danger_cases_df = dynamic_agent_df[
dynamic_agent_df["final_high_conf_wrong"] == True
].sort_values("final_max_prob", ascending=False)
danger_cases_df.head(20)
Out[71]:
| image_id | true_label | initial_pred_label | initial_max_prob | initial_correct | final_pred_label | final_max_prob | final_correct | asked_questions | num_questions | known_keys | changed_prediction | improved | worsened | still_wrong | still_correct | initial_high_conf_wrong | final_high_conf_wrong | unsafe_confidence_increase | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 218 | ISIC_0025394 | mel | nv | 0.736395 | False | nv | 0.999997 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 157 | ISIC_0030067 | bkl | nv | 0.740637 | False | nv | 0.999980 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 630 | ISIC_0027232 | nv | bkl | 0.662172 | False | bkl | 0.999977 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 15 | ISIC_0033899 | bkl | mel | 0.668489 | False | mel | 0.999973 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 141 | ISIC_0033460 | bkl | bkl | 0.750931 | True | mel | 0.999934 | False | [sex] | 1 | [sex] | True | False | True | False | False | False | True | False |
| 232 | ISIC_0034236 | mel | nv | 0.772531 | False | nv | 0.999867 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 417 | ISIC_0031520 | bcc | nv | 0.748447 | False | nv | 0.999862 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 1421 | ISIC_0032789 | nv | nv | 0.514896 | True | bkl | 0.999844 | False | [sex] | 1 | [sex] | True | False | True | False | False | False | True | False |
| 89 | ISIC_0024477 | bkl | mel | 0.768324 | False | mel | 0.999656 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 60 | ISIC_0031424 | bkl | mel | 0.430888 | False | mel | 0.999596 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 220 | ISIC_0026996 | mel | nv | 0.659830 | False | nv | 0.999496 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 344 | ISIC_0031957 | mel | nv | 0.796428 | False | nv | 0.998927 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 416 | ISIC_0029323 | bcc | nv | 0.600494 | False | nv | 0.998897 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 1445 | ISIC_0029860 | akiec | bcc | 0.612910 | False | bcc | 0.998831 | False | [age] | 1 | [age] | False | False | False | True | False | False | True | True |
| 11 | ISIC_0027957 | bkl | nv | 0.798028 | False | nv | 0.998547 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 1480 | ISIC_0031430 | akiec | bkl | 0.773255 | False | bkl | 0.998497 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 286 | ISIC_0033668 | mel | bkl | 0.475267 | False | bkl | 0.998449 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 198 | ISIC_0027253 | mel | nv | 0.707029 | False | nv | 0.998359 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 1457 | ISIC_0028335 | akiec | bkl | 0.714252 | False | bkl | 0.998214 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 180 | ISIC_0028346 | df | akiec | 0.646333 | False | akiec | 0.998203 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
In [72]:
# Cell 47: 看“问了以后变错”或者“问了以后更自信但仍然错”的病例
problem_cases_df = dynamic_agent_df[
(dynamic_agent_df["worsened"] == True) |
(dynamic_agent_df["unsafe_confidence_increase"] == True)
].sort_values("final_max_prob", ascending=False)
problem_cases_df.head(20)
Out[72]:
| image_id | true_label | initial_pred_label | initial_max_prob | initial_correct | final_pred_label | final_max_prob | final_correct | asked_questions | num_questions | known_keys | changed_prediction | improved | worsened | still_wrong | still_correct | initial_high_conf_wrong | final_high_conf_wrong | unsafe_confidence_increase | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 218 | ISIC_0025394 | mel | nv | 0.736395 | False | nv | 0.999997 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 157 | ISIC_0030067 | bkl | nv | 0.740637 | False | nv | 0.999980 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 630 | ISIC_0027232 | nv | bkl | 0.662172 | False | bkl | 0.999977 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 15 | ISIC_0033899 | bkl | mel | 0.668489 | False | mel | 0.999973 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 141 | ISIC_0033460 | bkl | bkl | 0.750931 | True | mel | 0.999934 | False | [sex] | 1 | [sex] | True | False | True | False | False | False | True | False |
| 232 | ISIC_0034236 | mel | nv | 0.772531 | False | nv | 0.999867 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 417 | ISIC_0031520 | bcc | nv | 0.748447 | False | nv | 0.999862 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 1421 | ISIC_0032789 | nv | nv | 0.514896 | True | bkl | 0.999844 | False | [sex] | 1 | [sex] | True | False | True | False | False | False | True | False |
| 89 | ISIC_0024477 | bkl | mel | 0.768324 | False | mel | 0.999656 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 60 | ISIC_0031424 | bkl | mel | 0.430888 | False | mel | 0.999596 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 220 | ISIC_0026996 | mel | nv | 0.659830 | False | nv | 0.999496 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 344 | ISIC_0031957 | mel | nv | 0.796428 | False | nv | 0.998927 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 416 | ISIC_0029323 | bcc | nv | 0.600494 | False | nv | 0.998897 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 1445 | ISIC_0029860 | akiec | bcc | 0.612910 | False | bcc | 0.998831 | False | [age] | 1 | [age] | False | False | False | True | False | False | True | True |
| 11 | ISIC_0027957 | bkl | nv | 0.798028 | False | nv | 0.998547 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 1480 | ISIC_0031430 | akiec | bkl | 0.773255 | False | bkl | 0.998497 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 286 | ISIC_0033668 | mel | bkl | 0.475267 | False | bkl | 0.998449 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 198 | ISIC_0027253 | mel | nv | 0.707029 | False | nv | 0.998359 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 1457 | ISIC_0028335 | akiec | bkl | 0.714252 | False | bkl | 0.998214 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
| 180 | ISIC_0028346 | df | akiec | 0.646333 | False | akiec | 0.998203 | False | [sex] | 1 | [sex] | False | False | False | True | False | False | True | True |
In [73]:
# Cell 48: 总结成一张表
dynamic_agent_summary = pd.DataFrame([{
"agent_name": "case_level_dynamic_agent",
"threshold": 0.80,
"max_questions": 2,
"avg_questions": dynamic_agent_df["num_questions"].mean(),
"initial_accuracy": dynamic_agent_df["initial_correct"].mean(),
"final_accuracy": dynamic_agent_df["final_correct"].mean(),
"changed_prediction_rate": dynamic_agent_df["changed_prediction"].mean(),
"improved_rate": dynamic_agent_df["improved"].mean(),
"worsened_rate": dynamic_agent_df["worsened"].mean(),
"initial_high_conf_wrong_rate": dynamic_agent_df["initial_high_conf_wrong"].mean(),
"final_high_conf_wrong_rate": dynamic_agent_df["final_high_conf_wrong"].mean(),
"unsafe_confidence_increase_rate": dynamic_agent_df["unsafe_confidence_increase"].mean(),
}])
dynamic_agent_summary
Out[73]:
| agent_name | threshold | max_questions | avg_questions | initial_accuracy | final_accuracy | changed_prediction_rate | improved_rate | worsened_rate | initial_high_conf_wrong_rate | final_high_conf_wrong_rate | unsafe_confidence_increase_rate | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | case_level_dynamic_agent | 0.8 | 2 | 0.369345 | 0.777178 | 0.782579 | 0.104659 | 0.04524 | 0.039838 | 0.057394 | 0.181634 | 0.116138 |
In [77]:
# Cell 49: 保存结果
from datetime import datetime
date_tag = datetime.now().strftime("%Y-%m-%d")
time_tag = datetime.now().strftime("%H%M%S")
dynamic_cases_path = SUPPORT_DIR / f"{date_tag}_{time_tag}_dynamic_agent_cases.csv"
dynamic_summary_path = SUPPORT_DIR / f"{date_tag}_{time_tag}_dynamic_agent_summary.csv"
danger_cases_path = SUPPORT_DIR / f"{date_tag}_{time_tag}_dynamic_agent_danger_cases.csv"
dynamic_agent_df.to_csv(dynamic_cases_path, index=False)
dynamic_agent_summary.to_csv(dynamic_summary_path, index=False)
danger_cases_df.to_csv(danger_cases_path, index=False)
print(dynamic_cases_path)
print(dynamic_summary_path)
print(danger_cases_path)
/Users/applesues01/Documents/Medical_Agent/supports/2026-08-04_104443_dynamic_agent_cases.csv /Users/applesues01/Documents/Medical_Agent/supports/2026-08-04_104443_dynamic_agent_summary.csv /Users/applesues01/Documents/Medical_Agent/supports/2026-08-04_104443_dynamic_agent_danger_cases.csv
问题不小啊感觉，sex疑似具有强烈误导性¶
The agent often became more confident after querying metadata, but this confidence increase did not necessarily correspond to improved correctness and sometimes produced unsafe high-confidence errors.
AI认为应添加的规则¶
规则 1：禁止优先问 sex¶
先把问题候选从： ["age", "sex", "location"] 改成： ["age", "location", "sex"] 或者更激进一点，第一版直接只允许： ["age", "location"]
规则 2：只有当新问题让“正确性代理”更好时才接受¶
当前我们只有 max_prob，这个太危险。 短期内最简单的安全约束是： 如果问完后 max_prob 虽然升高，但类别变化剧烈，先标记为风险 或者如果问完后进入某些已知高风险模式，就拒答
规则 3：加入拒答¶
比如： 如果最终 max_prob < 0.85，拒答 如果提问后 max_prob 暴涨，但前后类别变化过大，也拒答 如果问完后仍然不稳定，拒答
初始 image_only
低置信 -> 问
高置信 -> 不一定直接诊断，先过安全门
如果：最终置信度仍不足
或出现可疑高置信情况
或问题后仍冲突明显
-> abstain
In [78]:
# Cell 50: 先定义高风险类别
HIGH_RISK_LABELS = {"mel", "bcc", "akiec"}
HIGH_RISK_LABELS
Out[78]:
{'akiec', 'bcc', 'mel'}
In [79]:
# Cell 51: 计算 top1 / top2 / margin
def get_prediction_stats(pred_dict):
probs = np.array(pred_dict["prob_vector"])
sorted_probs = np.sort(probs)[::-1]
top1 = float(sorted_probs[0])
top2 = float(sorted_probs[1])
margin = float(top1 - top2)
return {
"top1": top1,
"top2": top2,
"margin": margin,
}
In [80]:
# Cell 52: 安全门规则
def safety_gate(pred_dict, confident_threshold=0.80, margin_threshold=0.20):
stats = get_prediction_stats(pred_dict)
pred_label = pred_dict["pred_label"]
top1 = stats["top1"]
top2 = stats["top2"]
margin = stats["margin"]
# 规则1：置信度不够，不能安全诊断
if top1 < confident_threshold:
return {
"safe": False,
"reason": "low_confidence",
"top1": top1,
"top2": top2,
"margin": margin,
}
# 规则2：前两类太接近，虽然top1高，但仍冲突明显
if margin < margin_threshold:
return {
"safe": False,
"reason": "small_margin",
"top1": top1,
"top2": top2,
"margin": margin,
}
# 规则3：高风险类别更保守
if pred_label in HIGH_RISK_LABELS and top1 < 0.90:
return {
"safe": False,
"reason": "high_risk_not_confident_enough",
"top1": top1,
"top2": top2,
"margin": margin,
}
return {
"safe": True,
"reason": "safe_to_diagnose",
"top1": top1,
"top2": top2,
"margin": margin,
}
这里解释一下这三个规则： low_confidence 不够自信，先别输出
small_margin 虽然第一名概率可能不低，但第二名咬得很紧，说明冲突还大
high_risk_not_confident_enough 如果预测的是高风险类，比如 mel，那我们更保守一点，要求更高置信度
In [81]:
# Cell 53: 先测试一下刚才那个危险病例
danger_pred = final_state["current_pred"]
danger_pred
Out[81]:
{'model_name': 'image_sex',
'pred_label': 'bkl',
'pred_idx': 2,
'max_prob': 0.9984972476959229,
'prob_vector': array([3.4605245e-07, 9.2998118e-07, 9.9849725e-01, 1.9880726e-07,
1.0359390e-07, 1.5011098e-03, 2.3849942e-08], dtype=float32)}
In [82]:
# Cell 54
safety_gate(danger_pred, confident_threshold=0.80, margin_threshold=0.20)
Out[82]:
{'safe': True,
'reason': 'safe_to_diagnose',
'top1': 0.9984972476959229,
'top2': 0.0015011098003014922,
'margin': 0.9969961378956214}
Stop Here！A deceiving Model！¶
The current dynamic agent can actively query case-specific metadata, but it may amplify erroneous beliefs into unsafe high-confidence predictions. A confidence-only safety gate is insufficient to prevent such failures.
In [ ]: