# Docker Desktop 4.47+（支持 macOS 13.7 + buildx + QEMU）
docker --version        # Docker version 28.x+
docker buildx version   # 必须支持 --platform linux/arm64

# 注册 QEMU 多架构支持（只需要执行一次，重启后需重新执行）
docker run --rm --privileged multiarch/qemu-user-static --reset -p yes

# 验证 ARM64 构建可用
docker buildx inspect --bootstrap
# 应看到 Platforms: linux/arm64, linux/amd64 等

FROM python:3.10-slim-bookworm          # ARM64 Python 基础镜像

# 环境变量预设离线和缓存路径
ENV PYTHONUNBUFFERED=1 \
    EVALSCOPE_CACHE=/workspace/cache/evalscope \
    MODELSCOPE_CACHE=/workspace/cache/modelscope/hub \
    HF_HOME=/workspace/cache/huggingface \
    HF_HUB_OFFLINE=1 \
    TRANSFORMERS_OFFLINE=1

# 安装构建依赖 + docker.io（SWE-bench 需要）+ git/git-lfs
RUN apt-get update && apt-get install -y --no-install-recommends \
        build-essential ca-certificates curl docker.io \
        git git-lfs jq openssh-client patch tini \
    && rm -rf /var/lib/apt/lists/*

# 安装 Python 依赖（evalscope + bfcl-eval + swebench + tau_bench）
COPY requirements.txt /workspace/requirements.txt
RUN pip install --upgrade pip && pip install -r /workspace/requirements.txt

# 拷入评测脚本和配置
COPY scripts /workspace/scripts
COPY configs /workspace/configs

ENTRYPOINT ["/usr/bin/tini", "--"]
CMD ["bash"]

evalscope[swe-bench,bfcl]==1.7.1    # EvalScope 核心 + SWE-bench + BFCL 插件
modelscope>=1.31.0                    # ModelScope SDK（下载数据集用）
datasets>=3.0.0                       # HuggingFace datasets 库
bfcl-eval==2025.10.27.1              # BFCL-v4 评测引擎（数据内置）
swebench==4.1.0                       # SWE-bench 评测引擎
git+https://github.com/sierra-research/tau-bench  # tau_bench 用户模拟

cd evalscope_offline
bash scripts/download_assets.sh

# 手动分步执行
python3 scripts/download_datasets.py --cache-dir modelscope_cache

docker buildx build --platform linux/arm64 \
  -t evalscope-offline:arm64 \
  -f Dockerfile.arm64 . --load

docker save evalscope-offline:arm64 | gzip -c > images/evalscope-offline-arm64.tar.gz

evalscope_offline/
├── Dockerfile.arm64                    # ARM64 镜像定义
├── requirements.txt                    # Python 依赖
├── modelscope_cache/                   # 数据集 (709MB) — 需要 scp 到 NPU
│   ├── cais/mmlu/
│   ├── evalscope/ceval/
│   ├── AI-ModelScope/gpqa_diamond/
│   ├── ZhipuAI/LongBench-v2/
│   ├── knoveleng/Minerva-Math/
│   ├── opencompass/humaneval/
│   └── evalscope/swe-bench-verified-mini/
├── images/
│   └── evalscope-offline-arm64.tar.gz  # Docker 镜像 (6GB) — 需要 scp 到 NPU
├── scripts/                            # 评测脚本 — scp 到 NPU
│   ├── run_eval.sh
│   ├── run_agent_eval.sh
│   └── run_smoke_mock_api.py
├── configs/                            # 参考配置
│   ├── 910b_eval_core.yaml
│   ├── 910b_eval_agent.yaml
│   └── 910b_eval_swe.yaml
└── wheels/                             # Python 离线安装包（可选）

1. images/evalscope-offline-arm64.tar.gz  (6.2GB)   Docker 镜像
2. modelscope_cache/                      (709MB)   数据集缓存
3. scripts/                                         评测脚本

# 从 Mac 传到 910B
scp images/evalscope-offline-arm64.tar.gz user@910b:/data/eval/
scp -r modelscope_cache/ user@910b:/data/eval/
scp -r scripts/ user@910b:/data/eval/

ssh user@910b
cd /data/eval
docker load -i evalscope-offline-arm64.tar.gz
# 验证
docker images evalscope-offline:arm64
# 输出: evalscope-offline:arm64 ... ~10GB

mkdir -p /data/eval/reports /data/eval/outputs

# 最终结构
/data/eval/
├── modelscope_cache/      # 核心数据集（不加 :ro）
├── reports/               # 评测报告输出
├── outputs/               # 中间结果
└── scripts/               # 评测脚本

curl http://127.0.0.1:8000/v1/models
# 预期返回: {"object":"list","data":[{"id":"你的模型名","object":"model"}]}

docker run --rm --network host \
  -e MODEL=qwen3-next \
  -e API_URL=http://10.7.6.39:3000/v1 \
  -e DATASETS="mmlu ceval gpqa_diamond" \
  -v /data/eval/modelscope_cache:/workspace/cache/modelscope \
  -v /data/eval/reports:/workspace/reports \
  evalscope-offline:arm64 \
  bash /workspace/scripts/run_eval.sh

docker run --rm --network host \
  -e MODEL=qwen3-next \
  -e API_URL=http://10.7.6.39:3000/v1 \
  -e DATASETS="minerva_math longbench_v2" \
  -v /data/eval/modelscope_cache:/workspace/cache/modelscope \
  -v /data/eval/reports:/workspace/reports \
  evalscope-offline:arm64 \
  bash /workspace/scripts/run_eval.sh

# HumanEval（基础代码生成）
docker run --rm --network host \
  -e MODEL=qwen3-next \
  -e API_URL=http://10.7.6.39:3000/v1 \
  -e DATASETS=humaneval \
  -v /data/eval/modelscope_cache:/workspace/cache/modelscope \
  -v /data/eval/reports:/workspace/reports \
  evalscope-offline:arm64 \
  bash /workspace/scripts/run_eval.sh

# SWE-bench（需要 Docker socket）
docker run --rm --network host \
  -e MODEL=qwen3-next \
  -e API_URL=http://10.7.6.39:3000/v1 \
  -e DATASETS=swe_bench_verified_mini \
  -e TIMEOUT=300 \
  -e MAX_TOKENS=4096 \
  -v /var/run/docker.sock:/var/run/docker.sock \
  -v /data/eval/modelscope_cache:/workspace/cache/modelscope \
  -v /data/eval/reports:/workspace/reports \
  evalscope-offline:arm64 \
  bash /workspace/scripts/run_eval.sh

# BFCL-v4（数据内置在镜像中，无需额外挂载）
docker run --rm --network host \
  -e MODEL=qwen3-next \
  -e API_URL=http://10.7.6.39:3000/v1 \
  -e DATASETS=bfcl_v4 \
  -v /data/eval/reports:/workspace/reports \
  evalscope-offline:arm64 \
  bash /workspace/scripts/run_agent_eval.sh

# tau_bench（需要用户模拟模型）
docker run --rm --network host \
  -e MODEL=qwen3-next \
  -e API_URL=http://10.7.6.39:3000/v1 \
  -e DATASETS=tau_bench \
  -e TAU_USER_MODEL=qwen3-next \
  -e TAU_USER_API_BASE=http://10.7.6.39:3000/v1 \
  -v /data/eval/reports:/workspace/reports \
  evalscope-offline:arm64 \
  bash /workspace/scripts/run_agent_eval.sh

docker run --rm --network host \
  -v /data/eval/modelscope_cache:/workspace/cache/modelscope \
  -v /data/eval/reports:/workspace/reports \
  evalscope-offline:arm64 \
  evalscope eval \
    --model qwen3-next \
    --api-url http://10.7.6.39:3000/v1 \
    --api-key ***"***\" \
    --eval-type openai_api \
    --datasets mmlu ceval \
    --limit 10 \
    --dataset-args '{\"mmlu\": {\"dataset_id\": \"/workspace/cache/modelscope/cais/mmlu\", \"few_shot_num\": 5}, \"ceval\": {\"dataset_id\": \"/workspace/cache/modelscope/evalscope/ceval\", \"few_shot_num\": 5}}' \
    --generation-config '{\"temperature\":0,\"max_tokens\":2048,\"timeout\":180}'\n```\n\n**注意：** `dataset_id` 参数直接指定本地路径（`local_path` 已废弃），不需要 `--dataset-hub` 和 `--dataset-dir`。

## 五、OpenAI 标准接口格式要求

EvalScope `openai_api` 模式对模型服务的要求如下。

### 5.1 GET /v1/models

```json
{
  "object": "list",
  "data": [{"id": "your-model-name", "object": "model"}]
}

{
  "id": "chatcmpl-xxx",
  "object": "chat.completion",
  "model": "your-model-name",
  "choices": [{
    "index": 0,
    "message": {"role": "assistant", "content": "回答内容"},
    "finish_reason": "stop"
  }],
  "usage": {"prompt_tokens": 100, "completion_tokens": 50, "total_tokens": 150}
}

{
  "choices": [{
    "message": {
      "role": "assistant",
      "content": null,
      "tool_calls": [{
        "id": "call_abc123",
        "type": "function",
        "function": {"name": "get_weather", "arguments": "{\"city\": \"Beijing\"}"}
      }]
    },
    "finish_reason": "tool_calls"
  }]
}

from fastapi import FastAPI
import httpx
import uuid

app = FastAPI()
YOUR_ENDPOINT = "http://127.0.0.1:9090/generate"

@app.get("/v1/models")
async def models():
    return {"object": "list", "data": [{"id": "my-model", "object": "model"}]}

@app.post("/v1/chat/completions")
async def chat(req: dict):
    messages = req.get("messages", [])
    tools = req.get("tools")
    prompt = messages[-1].get("content", "") if messages else ""
    if messages and messages[0].get("role") == "system":
        prompt = messages[0]["content"] + "\n\n" + prompt
    if tools:
        tool_desc = "\n".join(
            f"Function: {t['function']['name']}\n{t['function']['description']}"
            for t in tools
        )
        prompt = f"{tool_desc}\n\nUser request: {prompt}\n\nRespond with function call JSON."
    payload = {
        "prompt": prompt,
        "max_new_tokens": req.get("max_tokens", 2048),
        "temperature": req.get("temperature", 0),
    }
    async with httpx.AsyncClient() as client:
        resp = await client.post(YOUR_ENDPOINT, json=payload, timeout=300)
    result = resp.json()
    return {
        "id": f"chatcmpl-{uuid.uuid4().hex[:12]}",
        "object": "chat.completion",
        "model": req["model"],
        "choices": [{
            "index": 0,
            "message": {"role": "assistant", "content": result["text"]},
            "finish_reason": "stop"
        }],
        "usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
    }

pip install fastapi uvicorn httpx
uvicorn adapter:app --host 0.0.0.0 --port 8000

# 终端1：启动 mock API
docker run --rm -p 18080:18080 evalscope-offline:arm64 \
  python /workspace/scripts/run_smoke_mock_api.py --port 18080

# 终端2：冒烟测试
docker run --rm --network host \
  -e MODEL=mock \
  -e API_URL=http://127.0.0.1:18080/v1 \
  -e DATASETS=humaneval \
  -e LIMIT=1 \
  -v /data/eval/modelscope_cache:/workspace/cache/modelscope \
  evalscope-offline:arm64 \
  bash /workspace/scripts/run_eval.sh

--dataset-args '{"mmlu": {"dataset_id": "/path/to/data", "few_shot_num": 5}}'

WARNING: Retrying ... SSLError(SSLEOFError(8, '[SSL: UNEXPECTED_EOF_WHILE_READING] EOF occurred in violation of protocol'))
ERROR: Could not find a version that satisfies the requirement evalscope[swe-bench]>=1.8.0 (from versions: none)
ERROR: No matching distribution found for evalscope[swe-bench]>=1.8.0

RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple && \
    pip config set global.trusted-host pypi.tuna.tsinghua.edu.cn && \
    pip install --upgrade pip && pip install -r /workspace/requirements.txt

The conflict is caused by:
    datasets 2.18.0 depends on fsspec<=2024.2.0 and >=2023.1.0
    fsspec[http] 2024.2.0 depends on fsspec 2024.2.0
ERROR: ResolutionImpossible

curl -s https://pypi.org/pypi/modelscope/json | python3 -c \
 "import sys,json;[print(r) for r in json.load(sys.stdin)['info']['requires_dist'] if 'datasets' in r.lower()]"
# → datasets<=4.8.4,>=4.0.0; extra == "datasets"

datasets>=4.0.0,<=4.8.4

Cloning into '/workspace/tau-bench'...
fatal: unable to access 'https://github.com/sierra-research/tau-bench/':
  GnuTLS recv error (-110): The TLS connection was non-properly terminated.

# 宿主机（ghfast.top 是可用的 GitHub 加速镜像）
git clone --depth 1 https://ghfast.top/https://github.com/sierra-research/tau-bench tau-bench
rm -rf tau-bench/.git

# Dockerfile：由 git clone 改为 COPY（tau-bench 的 pip 依赖仍走清华源，正常）
COPY tau-bench /workspace/tau-bench
RUN pip install -e /workspace/tau-bench

docker buildx build --platform linux/arm64 \
  -f Dockerfile.arm64 -t evalscope-offline:arm64 --load .

docker image inspect evalscope-offline:arm64 --format '{{.Architecture}}'   # arm64
docker run --rm --platform linux/arm64 evalscope-offline:arm64 python -c \
 "import platform,evalscope,datasets,torch; \
  print(platform.machine(), evalscope.__version__, datasets.__version__, torch.__version__)"
# aarch64 1.9.0 4.8.4 2.12.1+cu130

osascript -e 'quit app "Docker"'
pkill -f com.docker.backend; pkill -f "Docker Desktop"; pkill -f com.docker.virtualization
open -a Docker

TypeError: Value.__init__() missing 1 required positional argument: 'dtype'

File "datasets/info.py", line 170, in __post_init__
  self.features = Features.from_dict(self.features)
File "datasets/features/features.py", line 1583, in generate_from_dict
  return class_type(**{k: v for k, v in obj.items() if k in field_names})
TypeError: Value.__init__() missing 1 required positional argument: 'dtype'

import datasets.features.features as f

_orig = f.generate_from_dict

def _patched(obj):
    if isinstance(obj, dict) and obj.get("_type") == "Value" and "dtype" not in obj:
        obj = {**obj, "dtype": "string"}
    return _orig(obj)

f.generate_from_dict = _patched

# install_patch.py
import shutil, site, os
src = '/workspace/scripts/patch_datasets_compat.py'
dst = os.path.join(site.getsitepackages()[0], 'patch_datasets_compat.py')
shutil.copy(src, dst)
with open(os.path.join(site.getsitepackages()[0], 'patch_datasets_compat.pth'), 'w') as f:
    f.write('import patch_datasets_compat\n')

from datasets.features.features import generate_from_dict
r = generate_from_dict({'_type': 'Value', 'id': None})
print(r)  # → Value('string')

# ❌ 错误：Docker 把第二个 import 当成了一条新指令
RUN python3 -c "
import shutil, site, os
...
"
# → dockerfile parse error: unknown instruction: import