CNI Plugin Bridge代码解析

在CNI所以默认提供的Plugin中，bridge应该算是最简单的插件了，针对IPAM Plugin，最简单的应该是host-local，这两个插件也是Kubernetes网络kubenet需要的两个插件。所以这里看一下这两个插件的代码。

所有官方维护的代码，都开源在containernetworking/plugins项目中了。

其中bridge的代码在plugins/main/bridge目录，最重要的是cmdAdd和cmdDel两个函数，对应CNI SPEC中的ADD和DEL两个主要操作。主要来看一下cmdAdd的实现，精简（删除一些错误处理）后的代码如下：

func cmdAdd(args *skel.CmdArgs) error {
	n, cniVersion, err := loadNetConf(args.StdinData)

	if n.IsDefaultGW {
		n.IsGW = true
	}

	br, brInterface, err := setupBridge(n)
	if err != nil {
		return err
	}

	netns, err := ns.GetNS(args.Netns)
	defer netns.Close()

	hostInterface, containerInterface, err := setupVeth(netns, br, args.IfName, n.MTU, n.HairpinMode)

	r, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData)
	if err != nil {
		return err
	}

	result, err := current.NewResultFromResult(r)

	if len(result.IPs) == 0 {
		return errors.New("IPAM plugin returned missing IP config")
	}

	result.Interfaces = []*current.Interface{brInterface, hostInterface, containerInterface}

	gwsV4, gwsV6, err := calcGateways(result, n)
	if err != nil {
		return err
	}

	if err := netns.Do(func(_ ns.NetNS) error {
		contVeth, err := net.InterfaceByName(args.IfName)

		for _, ipc := range result.IPs {
			if ipc.Version == "6" && (n.HairpinMode || n.PromiscMode) {
				if err := disableIPV6DAD(args.IfName); err != nil {
					return err
				}
				break
			}
		}

		if err := ipam.ConfigureIface(args.IfName, result); err != nil {
			return err
		}

		for _, ipc := range result.IPs {
			if ipc.Version == "4" {
				_ = arping.GratuitousArpOverIface(ipc.Address.IP, *contVeth)
			}
		}
		return nil
	}); err != nil {
		return err
	}

	if n.IsGW {
		var firstV4Addr net.IP
		for _, gws := range []*gwInfo{gwsV4, gwsV6} {
			for _, gw := range gws.gws {
				if gw.IP.To4() != nil && firstV4Addr == nil {
					firstV4Addr = gw.IP
				}

				err = ensureBridgeAddr(br, gws.family, &gw, n.ForceAddress)
				if err != nil {
					return fmt.Errorf("failed to set bridge addr: %v", err)
				}
			}

			if gws.gws != nil {
				if err = enableIPForward(gws.family); err != nil {
					return fmt.Errorf("failed to enable forwarding: %v", err)
				}
			}
		}
	}

	if n.IPMasq {
		chain := utils.FormatChainName(n.Name, args.ContainerID)
		comment := utils.FormatComment(n.Name, args.ContainerID)
		for _, ipc := range result.IPs {
			if err = ip.SetupIPMasq(ip.Network(&ipc.Address), chain, comment); err != nil {
				return err
			}
		}
	}

	br, err = bridgeByName(n.BrName)

	brInterface.Mac = br.Attrs().HardwareAddr.String()

	result.DNS = n.DNS

	return types.PrintResult(result, cniVersion)
}

主要的就是几大步，首先，调用setupBridge确保机器上有对应的bridge，然后再调用setupVeth在容器对应的namespace下创建好虚拟网络接口。创建完成，就需要通过ipam.ExecAdd(n.IPAM.Type, args.StdinData)向IPAM插件获取IP地址了。
拿到了IP地址，就会根据IP地址计算对应的路由和网关，然后调用ipam.ConfigureIface(args.IfName, result)将IP地址设置到对应的虚拟网络接口上，同时，还需要将主机上的bridge加上网关的IP，并且开启主机的ip_forward,
最后再调用ip.SetupIPMasq(ip.Network(&ipc.Address), chain, comment)加上IP转发规则，一切正常，按要求输出结果到stdout整个Add操作结束。

而对于IPAM host-local插件，其代码在plugins/ipam/host-local目录下，主要逻辑同样比较简单，精简后的cmdAdd函数如下：

func cmdAdd(args *skel.CmdArgs) error {
	ipamConf, confVersion, err := allocator.LoadIPAMConfig(args.StdinData, args.Args)

	result := &current.Result{}

	if ipamConf.ResolvConf != "" {
		dns, err := parseResolvConf(ipamConf.ResolvConf)
		if err != nil {
			return err
		}
		result.DNS = *dns
	}

	store, err := disk.New(ipamConf.Name, ipamConf.DataDir)
	defer store.Close()

	allocs := []*allocator.IPAllocator{}

	requestedIPs := map[string]net.IP{} //net.IP cannot be a key

	for _, ip := range ipamConf.IPArgs {
		requestedIPs[ip.String()] = ip
	}

	for idx, rangeset := range ipamConf.Ranges {
		allocator := allocator.NewIPAllocator(&rangeset, store, idx)

		// Check to see if there are any custom IPs requested in this range.
		var requestedIP net.IP
		for k, ip := range requestedIPs {
			if rangeset.Contains(ip) {
				requestedIP = ip
				delete(requestedIPs, k)
				break
			}
		}

		ipConf, err := allocator.Get(args.ContainerID, requestedIP)
		if err != nil {
			// Deallocate all already allocated IPs
			for _, alloc := range allocs {
				_ = alloc.Release(args.ContainerID)
			}
			return fmt.Errorf("failed to allocate for range %d: %v", idx, err)
		}

		allocs = append(allocs, allocator)

		result.IPs = append(result.IPs, ipConf)
	}

	// If an IP was requested that wasn't fulfilled, fail
	if len(requestedIPs) != 0 {
		for _, alloc := range allocs {
			_ = alloc.Release(args.ContainerID)
		}
		errstr := "failed to allocate all requested IPs:"
		for _, ip := range requestedIPs {
			errstr = errstr + " " + ip.String()
		}
		return fmt.Errorf(errstr)
	}

	result.Routes = ipamConf.Routes

	return types.PrintResult(result, confVersion)
}

主要逻辑就是allocator.LoadIPAMConfig读取IPAM的配置，得到需要分配的IP段，然后针对每个IP段，调用allocator.Get获取IP地址，再做一些相应的判断等操作，最后输出结果。

其中allocator需要一个Store实例用来存储已经分配的IP地址信息，默认使用的是plugins/ipam/host-local/backend/disk/backend.go将信息存到磁盘上，实现很简单，就是每个IP存一个文件，文件内容就是ContainerID。

有bridge和host-local配合，kubenet就可以正常工作了，但是如果引入更复杂的网络结构，就需要使用更复杂的CNI插件了。