android中eBPF 流量监控

简介

eBPF（Extended Berkeley Packet Filter ）可以在不需要修改内核代码情况下，动态加载程序对内核进行扩展。eBPF网络流量工具使用内核和用户空间实现的组合来监视自上次设备启动以来设备上的网络使用情况，Android9开始使用了该功能来进行流量监控。

预置条件

1. 内核中需要打开以下配置

CONFIG_CGROUP_BPF=y
CONFIG_BPF=y
CONFIG_BPF_SYSCALL=y
CONFIG_NETFILTER_XT_MATCH_BPF=y
CONFIG_INET_UDP_DIAG=y

1. MEM_LOCK限制

设备MEM_LOCK rlimit 必须设置为 8 MB 或更多。

加载程序

代码位于('/system/netd/bpfloader/')中。
定义map:

    DECLARE_MAP(cookieTagMap, COOKIE_TAG_MAP_PATH);
    DECLARE_MAP(uidCounterSetMap, UID_COUNTERSET_MAP_PATH);
    DECLARE_MAP(appUidStatsMap, APP_UID_STATS_MAP_PATH);
    DECLARE_MAP(uidStatsMap, UID_STATS_MAP_PATH);
    DECLARE_MAP(tagStatsMap, TAG_STATS_MAP_PATH);
    DECLARE_MAP(ifaceStatsMap, IFACE_STATS_MAP_PATH);
    DECLARE_MAP(dozableUidMap, DOZABLE_UID_MAP_PATH);
    DECLARE_MAP(standbyUidMap, STANDBY_UID_MAP_PATH);
    DECLARE_MAP(powerSaveUidMap, POWERSAVE_UID_MAP_PATH);

每个map的内存空间大小计算公式（BpfUtils.h中有说明）：

 elem_size = 40 + roundup(key_size, 8) + roundup value_size, 8)
 cost = roundup_pow_of_two(max_entries) * 16 + elem_size * max_entries + elem_size * number_of_CPU

之后会根据传入的参数类型进行参数配置：

 switch (opt) {
            case 'i':
                doIngress = true;
                break;
            case 'e':
                doEgress = true;
                break;
            case 'p':
                doPrerouting = true;
                break;
            case 'm':
                doMangle = true;
                break;
            default:
                usage();
                FAIL("unknown argument %c", opt);

netd中支持以下类型的参数：

    ALOGE( "Usage: ./bpfloader [-i] [-e]\n"
           "   -i load ingress bpf program\n"
           "   -e load egress bpf program\n"
           "   -p load prerouting xt_bpf program\n"
           "   -m load mangle xt_bpf program\n");

加载"/bpf_kern.o"文件。
android::bpf::parseProgramsFromFile(BPF_PROG_SRC);

根据不同的传入参数进行attach操作,attach指定把bpf程序hook到哪个内核监控点上。

 if (doIngress) {
        ret = loadAndAttachProgram(BPF_CGROUP_INET_INGRESS, BPF_INGRESS_PROG_PATH,
                                   BPF_CGROUP_INGRESS_PROG_NAME, mapPatterns);
        if (ret) {
            FAIL("Failed to set up ingress program");
        }
    }
    if (doEgress) {
        ret = loadAndAttachProgram(BPF_CGROUP_INET_EGRESS, BPF_EGRESS_PROG_PATH,
                                   BPF_CGROUP_EGRESS_PROG_NAME, mapPatterns);
        if (ret) {
            FAIL("Failed to set up ingress program");
        }
    }
    if (doPrerouting) {
        ret = loadAndAttachProgram(MAX_BPF_ATTACH_TYPE, XT_BPF_INGRESS_PROG_PATH,
                                   XT_BPF_INGRESS_PROG_NAME, mapPatterns);
        if (ret) {
            FAIL("Failed to set up xt_bpf program");
        }
    }
    if (doMangle) {
        ret = loadAndAttachProgram(MAX_BPF_ATTACH_TYPE, XT_BPF_EGRESS_PROG_PATH,
                                   XT_BPF_EGRESS_PROG_NAME, mapPatterns);
        if (ret) {
            FAIL("Failed to set up xt_bpf program");
        }
    }

测试验证

BpfNetworkStatsTest可以对bpf的功能进行测试:

 void SetUp() {
        mFakeCookieTagMap = BpfMap<uint64_t, UidTag>(createMap(
            BPF_MAP_TYPE_HASH, sizeof(uint64_t), sizeof(struct UidTag), TEST_MAP_SIZE, 0));
        ASSERT_LE(0, mFakeCookieTagMap.getMap());

        mFakeAppUidStatsMap = BpfMap<uint32_t, StatsValue>(createMap(
            BPF_MAP_TYPE_HASH, sizeof(uint32_t), sizeof(struct StatsValue), TEST_MAP_SIZE, 0));
        ASSERT_LE(0, mFakeAppUidStatsMap.getMap());
        ....
        mFakeIfaceStatsMap = BpfMap<uint32_t, StatsValue>(createMap(
            BPF_MAP_TYPE_HASH, sizeof(uint32_t), sizeof(struct StatsValue), TEST_MAP_SIZE, 0));
        ASSERT_LE(0, mFakeIfaceStatsMap.getMap());
    }
    ...
    TEST_F(BpfNetworkStatsHelperTest, TestGetUidStatsTotal) {
    SKIP_IF_BPF_NOT_SUPPORTED;

    updateIfaceMap(IFACE_NAME1, IFACE_INDEX1);
    updateIfaceMap(IFACE_NAME2, IFACE_INDEX2);
    updateIfaceMap(IFACE_NAME3, IFACE_INDEX3);
    StatsValue value1 = {.rxBytes = TEST_BYTES0,
                         .rxPackets = TEST_PACKET0,
                         .txBytes = TEST_BYTES1,
                         .txPackets = TEST_PACKET1,};
    StatsValue value2 = {
        .rxBytes = TEST_BYTES0 * 2,
        .rxPackets = TEST_PACKET0 * 2,
        .txBytes = TEST_BYTES1 * 2,
        .txPackets = TEST_PACKET1 * 2,
    };
    ASSERT_TRUE(isOk(mFakeAppUidStatsMap.writeValue(TEST_UID1, value1, BPF_ANY)));
    ASSERT_TRUE(isOk(mFakeAppUidStatsMap.writeValue(TEST_UID2, value2, BPF_ANY)));
    ...
    ASSERT_EQ((unsigned long)1, lines.size());
    expectStatsLineEqual(value1, IFACE_NAME3, TEST_UID2, TEST_COUNTERSET1, 0, lines.front());
}

其他

在google官网的ebpf-traffic-monitor文章中可以看到相关vts和cts测试信息。