摘要：假设你对kube-proxy的工作原理有一定的了解。本文基于kubernetes v1.5代码对kube-proxy的源代码文件夹结构进行了分析，并以iptables mode为例进行了完整流程的源代码分析，给出了其内部实现的模块逻辑图，希望对你深入理解kube-proxy有所帮助。

kube-proxy介绍

请參考我的还有一篇博文：kube-proxy工作原理

源代码文件夹结构分析

  1. cmd/kube-proxy      //负责kube-proxy的创建，启动的入口
  2. .
  3. ├── app
  4. │   ├── conntrack.go    //linux kernel的nf_conntrack-sysctl的interface定义，很多其它关于conntracker的定义请看https://www.kernel.org/doc/Documentation/networking/nf_conntrack-sysctl.txt
  5. │   ├── options
  6. │   │   └── options.go    //kube-proxy的參数定义ProxyServerConfig及相关方法
  7. │   ├── server.go    //ProxyServer结构定义及其创建(NewProxyServerDefault)和运行(Run)的方法。
  8. │   └── server_test.go
  9. └── proxy.go    //kube-proxy的main方法
  11. pkg/proxy
  12. .
  13. ├── OWNERS
  14. ├── config
  15. │   ├── api.go    //给proxy配置Service和Endpoint的Reflectors和Cache.Store
  16. │   ├── api_test.go
  17. │   ├── config.go    //定义ServiceUpdate。EndpointUpdate结构体以及ServiceConfigHandler，EndpointConfigHandler来处理Service和Endpoint的Update
  18. │   ├── config_test.go
  19. │   └── doc.go
  20. ├── doc.go
  21. ├── healthcheck    //负责service listener和endpoint的health check，add/delete请求。
  22. │   ├── api.go
  23. │   ├── doc.go
  24. │   ├── healthcheck.go
  25. │   ├── healthcheck_test.go
  26. │   ├── http.go
  27. │   ├── listener.go
  28. │   └── worker.go
  29. ├── iptables    //proxy mode为iptables的实现
  30. │   ├── proxier.go
  31. │   └── proxier_test.go
  32. ├── types.go
  33. ├── userspace    //proxy mode为userspace的实现
  34. │   ├── loadbalancer.go
  35. │   ├── port_allocator.go
  36. │   ├── port_allocator_test.go
  37. │   ├── proxier.go
  38. │   ├── proxier_test.go
  39. │   ├── proxysocket.go
  40. │   ├── rlimit.go
  41. │   ├── rlimit_windows.go
  42. │   ├── roundrobin.go
  43. │   ├── roundrobin_test.go
  44. │   └── udp_server.go
  45. └── winuserspace    //windows OS时。proxy mode为userspace的实现
  46.     ├── loadbalancer.go
  47.     ├── port_allocator.go
  48.     ├── port_allocator_test.go
  49.     ├── proxier.go
  50.     ├── proxier_test.go
  51.     ├── proxysocket.go
  52.     ├── roundrobin.go
  53.     ├── roundrobin_test.go
  54.     └── udp_server.go

内部实现模块逻辑图

源代码分析

main

kube-proxy的main入口在：cmd/kube-proxy/proxy.go:39

  1. func main() {
  2.     //创建kube-proxy的默认config对象
  3.     config := options.NewProxyConfig()
  4.     //用kube-proxy命令行的參数替换默认參数
  5.     config.AddFlags(pflag.CommandLine)
  7.     flag.InitFlags()
  8.     logs.InitLogs()
  9.     defer logs.FlushLogs()
  11.     verflag.PrintAndExitIfRequested()
  13.     //依据config创建ProxyServer
  14.     s, err := app.NewProxyServerDefault(config)
  15.     if err != nil {
  16.         fmt.Fprintf(os.Stderr, "%v\n", err)
  17.         os.Exit(1)
  18.     }
  20.     //运行Run方法让kube-proxy開始干活了
  21.     if err = s.Run(); err != nil {
  22.         fmt.Fprintf(os.Stderr, "%v\n", err)
  23.         os.Exit(1)
  24.     }
  25. }

main方法中。我们重点关注app.NewProxyServerDefault(config)创建ProxyServer和Run方法。

创建ProxyServer

NewProxyServerDefault负责依据提供的config參数创建一个新的ProxyServer对象，其代码比較长，逻辑相对复杂，以下会挑重点说一下。

  1. cmd/kube-proxy/app/server.go:131
  3. func NewProxyServerDefault(config *options.ProxyServerConfig) (*ProxyServer, error) {
  4.     ...
  6.     // Create a iptables utils.
  7.     execer := exec.New()
  9.     if runtime.GOOS == "windows" {
  10.         netshInterface = utilnetsh.New(execer)
  11.     } else {
  12.         dbus = utildbus.New()
  13.         iptInterface = utiliptables.New(execer, dbus, protocol)
  14.     }
  16.     ...
  17.     //设置OOM_SCORE_ADJ
  18.     var oomAdjuster *oom.OOMAdjuster
  19.     if config.OOMScoreAdj != nil {
  20.         oomAdjuster = oom.NewOOMAdjuster()
  21.         if err := oomAdjuster.ApplyOOMScoreAdj(0, int(*config.OOMScoreAdj)); err != nil {
  22.             glog.V(2).Info(err)
  23.         }
  24.     }
  26.     ...
  28.     // Create a Kube Client
  29.     ...
  31.     // 创建event Broadcaster和event recorder
  32.     hostname := nodeutil.GetHostname(config.HostnameOverride)
  33.     eventBroadcaster := record.NewBroadcaster()
  34.     recorder := eventBroadcaster.NewRecorder(v1.EventSource{Component: "kube-proxy", Host: hostname})
  36.     //定义proxier和endpointsHandler。分别用于处理services和endpoints的update event。
  37.     var proxier proxy.ProxyProvider
  38.     var endpointsHandler proxyconfig.EndpointsConfigHandler
  40.     //从config中获取proxy mode
  41.     proxyMode := getProxyMode(string(config.Mode), client.Core().Nodes(), hostname, iptInterface, iptables.LinuxKernelCompatTester{})
  43.     // proxy mode为iptables场景
  44.     if proxyMode == proxyModeIPTables {
  45.         glog.V(0).Info("Using iptables Proxier.")
  46.         if config.IPTablesMasqueradeBit == nil {
  47.             // IPTablesMasqueradeBit must be specified or defaulted.
  48.             return nil, fmt.Errorf("Unable to read IPTablesMasqueradeBit from config")
  49.         }
  51.         //调用pkg/proxy/iptables/proxier.go:222中的iptables.NewProxier来创建proxier，赋值给前面定义的proxier和endpointsHandler。表示由该proxier同一时候负责service和endpoint的event处理。
  52.         proxierIPTables, err := iptables.NewProxier(iptInterface, utilsysctl.New(), execer, config.IPTablesSyncPeriod.Duration, config.IPTablesMinSyncPeriod.Duration, config.MasqueradeAll, int(*config.IPTablesMasqueradeBit), config.ClusterCIDR, hostname, getNodeIP(client, hostname))
  53.         if err != nil {
  54.             glog.Fatalf("Unable to create proxier: %v", err)
  55.         }
  56.         proxier = proxierIPTables
  57.         endpointsHandler = proxierIPTables
  58.         // No turning back. Remove artifacts that might still exist from the userspace Proxier.
  59.         glog.V(0).Info("Tearing down userspace rules.")
  60.         userspace.CleanupLeftovers(iptInterface)
  61.     }
  62.     // proxy mode为userspace场景
  63.     else {
  64.         glog.V(0).Info("Using userspace Proxier.")
  65.         // This is a proxy.LoadBalancer which NewProxier needs but has methods we don't need for
  66.         // our config.EndpointsConfigHandler.
  67.         loadBalancer := userspace.NewLoadBalancerRR()
  68.         // set EndpointsConfigHandler to our loadBalancer
  69.         endpointsHandler = loadBalancer
  71.         var proxierUserspace proxy.ProxyProvider
  73.         // windows OS场景下，调用pkg/proxy/winuserspace/proxier.go:146的winuserspace.NewProxier来创建proxier。
  74.         if runtime.GOOS == "windows" {
  75.             proxierUserspace, err = winuserspace.NewProxier(
  76.                 loadBalancer,
  77.                 net.ParseIP(config.BindAddress),
  78.                 netshInterface,
  79.                 *utilnet.ParsePortRangeOrDie(config.PortRange),
  80.                 // TODO @pires replace below with default values, if applicable
  81.                 config.IPTablesSyncPeriod.Duration,
  82.                 config.UDPIdleTimeout.Duration,
  83.             )
  84.         } 
  86.         // linux OS场景下，调用pkg/proxy/userspace/proxier.go:143的userspace.NewProxier来创建proxier。
  87.         else {
  88.             proxierUserspace, err = userspace.NewProxier(
  89.                 loadBalancer,
  90.                 net.ParseIP(config.BindAddress),
  91.                 iptInterface,
  92.                 *utilnet.ParsePortRangeOrDie(config.PortRange),
  93.                 config.IPTablesSyncPeriod.Duration,
  94.                 config.IPTablesMinSyncPeriod.Duration,
  95.                 config.UDPIdleTimeout.Duration,
  96.             )
  97.         }
  98.         if err != nil {
  99.             glog.Fatalf("Unable to create proxier: %v", err)
  100.         }
  101.         proxier = proxierUserspace
  102.         // Remove artifacts from the pure-iptables Proxier, if not on Windows.
  103.         if runtime.GOOS != "windows" {
  104.             glog.V(0).Info("Tearing down pure-iptables proxy rules.")
  105.             iptables.CleanupLeftovers(iptInterface)
  106.         }
  107.     }
  109.     // Add iptables reload function, if not on Windows.
  110.     if runtime.GOOS != "windows" {
  111.         iptInterface.AddReloadFunc(proxier.Sync)
  112.     }
  114.     // Create configs (i.e. Watches for Services and Endpoints)
  115.     // 创建serviceConfig负责service的watchforUpdates
  116.     serviceConfig := proxyconfig.NewServiceConfig()
  118.     //给serviceConfig注冊proxier，既加入相应的listener用来处理service update时逻辑。
  119.     serviceConfig.RegisterHandler(proxier)
  121.     // 创建endpointsConfig负责endpoint的watchforUpdates
  122.     endpointsConfig := proxyconfig.NewEndpointsConfig()
  124.     //给endpointsConfig注冊endpointsHandler，既加入相应的listener用来处理endpoint update时的逻辑。
  125.     endpointsConfig.RegisterHandler(endpointsHandler)
  127.     //NewSourceAPI creates config source that watches for changes to the services and endpoints.
  128.     //NewSourceAPI通过ListWatch apiserver的Service和endpoint，并周期性的维护serviceStore和endpointStore的更新
  129.     proxyconfig.NewSourceAPI(
  130.         client.Core().RESTClient(),
  131.         config.ConfigSyncPeriod,
  132.         serviceConfig.Channel("api"), //Service Update Channel
  133.         endpointsConfig.Channel("api"),  //endpoint update channel
  134.     )
  136.     ...
  138.     //把前面创建的对象作为參数。构造出ProxyServer对象。
  139.     return NewProxyServer(client, config, iptInterface, proxier, eventBroadcaster, recorder, conntracker, proxyMode)
  140. }

NewProxyServerDefault中的核心逻辑我都已经在上述代码中加入了凝视，当中有几个地方须要我们再深入跟进去看看：proxyconfig.NewServiceConfig。proxyconfig.NewEndpointsConfig，serviceConfig.RegisterHandler，endpointsConfig.RegisterHandler。proxyconfig.NewSourceAPI。

proxyconfig.NewServiceConfig

我们对ServiceConfig的代码分析一遍，EndpointsConfig的代码则相似。

  1. pkg/proxy/config/config.go:192
  2. func NewServiceConfig() *ServiceConfig {
  3.     // 创建updates channel
  4.     updates := make(chan struct{}, 1)
  6.     // 构建serviceStore对象
  7.     store := &serviceStore{updates: updates, services: make(map[string]map[types.NamespacedName]api.Service)}
  8.     mux := config.NewMux(store)
  10.     // 新建Broadcaster。在兴许的serviceConfig.RegisterHandler会注冊该Broadcaster的listener。
  11.     bcaster := config.NewBroadcaster()
  13.     //启动协程，立即開始watch updates channel
  14.     go watchForUpdates(bcaster, store, updates)
  16.     return &ServiceConfig{mux, bcaster, store}
  17. }

以下我们再跟进watchForUpdates去看看。

  1. pkg/proxy/config/config.go:292
  2. func watchForUpdates(bcaster *config.Broadcaster, accessor config.Accessor, updates <-chan struct{}) {
  3.     for true {
  4.         <-updates
  5.         bcaster.Notify(accessor.MergedState())
  6.     }
  7. }

watchForUpdates就是一直在watch updates channel，假设有数据，则立马由该Broadcaster Notify到注冊的listeners。

Notify的代码例如以下。可见。它负责将数据通知给全部的listener。并调用各个listener的OnUpdate方法。

  1. pkg/util/config/config.go:133
  2. // Notify notifies all listeners.
  3. func (b *Broadcaster) Notify(instance interface{}) {
  4.     b.listenerLock.RLock()
  5.     listeners := b.listeners
  6.     b.listenerLock.RUnlock()
  7.     for _, listener := range listeners {
  8.         listener.OnUpdate(instance)
  9.     }
  10. }
  12. func (f ListenerFunc) OnUpdate(instance interface{}) {
  13.     f(instance)
  14. }

serviceConfig.RegisterHandler

上面分析的proxyconfig.NewServiceConfig负责创建ServiceConfig。開始watch updates channel了，当从channel中取到值的时候，Broadcaster就会通知listener进行处理。serviceConfig.RegisterHandler正是负责给Broadcaster注冊listener的。其代码例如以下。

  1. pkg/proxy/config/config.go:205
  3. func (c *ServiceConfig) RegisterHandler(handler ServiceConfigHandler) {
  4.     //给ServiceConfig的Broadcaster注冊listener。
  5.     c.bcaster.Add(config.ListenerFunc(func(instance interface{}) {
  6.         glog.V(3).Infof("Calling handler.OnServiceUpdate()")
  7.         handler.OnServiceUpdate(instance.([]api.Service))
  8.     }))
  9. }

上面分析proxyconfig.NewServiceConfig时可知，当从updates channel中取到值的时候，终于会调用相应的ListenerFunc(instance)进行处理。在这里。也就是调用:

  1.         glog.V(3).Infof("Calling handler.OnServiceUpdate()")
  2.         handler.OnServiceUpdate(instance.([]api.Service))
  3.     }

即调用到handler.OnServiceUpdate。每种proxymode相应的proxier都有相应的handler.OnServiceUpdate接口实现，我们以iptables mode为例。看看handler.OnServiceUpdate的实现：

  1. pkg/proxy/iptables/proxier.go:428
  2. func (proxier *Proxier) OnServiceUpdate(allServices []api.Service) {
  3.     ...
  5.     proxier.syncProxyRules()
  6.     proxier.deleteServiceConnections(staleUDPServices.List())
  8. }

因此，终于关键的逻辑都转向了proxier.syncProxyRules()，从我们上面给出的内部模块交互图也能看得出来。对于proxier.syncProxyRules()。我们放到后面来具体讨论，如今你仅仅要知道proxier.syncProxyRules()负责将proxy中缓存的service/endpoint同步更新到iptables中生成相应Chain和NAT Rules。

proxyconfig.NewEndpointsConfig

endpointsConfig的逻辑和serviceConfig的相似，在这里仅仅给出相应代码。不再跟进分析。

  1. pkg/proxy/config/config.go:84
  3. func NewEndpointsConfig() *EndpointsConfig {
  4.     // The updates channel is used to send interrupts to the Endpoints handler.
  5.     // It's buffered because we never want to block for as long as there is a
  6.     // pending interrupt, but don't want to drop them if the handler is doing
  7.     // work.
  8.     updates := make(chan struct{}, 1)
  9.     store := &endpointsStore{updates: updates, endpoints: make(map[string]map[types.NamespacedName]api.Endpoints)}
  10.     mux := config.NewMux(store)
  11.     bcaster := config.NewBroadcaster()
  12.     go watchForUpdates(bcaster, store, updates)
  13.     return &EndpointsConfig{mux, bcaster, store}
  14. }

endpointsConfig.RegisterHandler

  1. pkg/proxy/config/config.go:97
  3. func (c *EndpointsConfig) RegisterHandler(handler EndpointsConfigHandler) {
  4.     c.bcaster.Add(config.ListenerFunc(func(instance interface{}) {
  5.         glog.V(3).Infof("Calling handler.OnEndpointsUpdate()")
  6.         handler.OnEndpointsUpdate(instance.([]api.Endpoints))
  7.     }))
  8. }

proxyconfig.NewSourceAPI

proxyconfig.NewSourceAPI是非常关键的，它负责给service updates channel和endpoint updates channel配置数据源，它是通过周期性的List和Watch kube-apiserver中的all service and endpoint来提供数据的。发给相应的channel。

默认的List周期是15min。可通过--config-sync-period改动。以下来看其具体代码：

  1. func NewSourceAPI(c cache.Getter, period time.Duration, servicesChan chan<- ServiceUpdate, endpointsChan chan<- EndpointsUpdate) {
  2.     servicesLW := cache.NewListWatchFromClient(c, "services", api.NamespaceAll, fields.Everything())
  3.     cache.NewReflector(servicesLW, &api.Service{}, NewServiceStore(nil, servicesChan), period).Run()
  5.     endpointsLW := cache.NewListWatchFromClient(c, "endpoints", api.NamespaceAll, fields.Everything())
  6.     cache.NewReflector(endpointsLW, &api.Endpoints{}, NewEndpointsStore(nil, endpointsChan), period).Run()
  7. }
  9. // NewServiceStore creates an undelta store that expands updates to the store into
  10. // ServiceUpdate events on the channel. If no store is passed, a default store will
  11. // be initialized. Allows reuse of a cache store across multiple components.
  12. func NewServiceStore(store cache.Store, ch chan<- ServiceUpdate) cache.Store {
  13.     fn := func(objs []interface{}) {
  14.         var services []api.Service
  15.         for _, o := range objs {
  16.             services = append(services, *(o.(*api.Service)))
  17.         }
  18.         ch <- ServiceUpdate{Op: SET, Services: services}
  19.     }
  20.     if store == nil {
  21.         store = cache.NewStore(cache.MetaNamespaceKeyFunc)
  22.     }
  23.     return &cache.UndeltaStore{
  24.         Store:    store,
  25.         PushFunc: fn,
  26.     }
  27. }
  29. // NewEndpointsStore creates an undelta store that expands updates to the store into
  30. // EndpointsUpdate events on the channel. If no store is passed, a default store will
  31. // be initialized. Allows reuse of a cache store across multiple components.
  32. func NewEndpointsStore(store cache.Store, ch chan<- EndpointsUpdate) cache.Store {
  33.     fn := func(objs []interface{}) {
  34.         var endpoints []api.Endpoints
  35.         for _, o := range objs {
  36.             endpoints = append(endpoints, *(o.(*api.Endpoints)))
  37.         }
  38.         ch <- EndpointsUpdate{Op: SET, Endpoints: endpoints}
  39.     }
  40.     if store == nil {
  41.         store = cache.NewStore(cache.MetaNamespaceKeyFunc)
  42.     }
  43.     return &cache.UndeltaStore{
  44.         Store:    store,
  45.         PushFunc: fn,
  46.     }
  47. }

代码非常easy。不须要过多解释。

运行Run開始工作

创建完ProxyServer后，就运行Run方法開始工作了,它主要负责周期性(default 30s)的同步proxy中的services/endpionts到iptables中生成相应Chain and NAT Rules。

  1. cmd/kube-proxy/app/server.go:308
  2. func (s *ProxyServer) Run() error {
  3.     ...
  5.     // Start up a webserver if requested
  6.     if s.Config.HealthzPort > 0 {
  7.         http.HandleFunc("/proxyMode", func(w http.ResponseWriter, r *http.Request) {
  8.             fmt.Fprintf(w, "%s", s.ProxyMode)
  9.         })
  10.         configz.InstallHandler(http.DefaultServeMux)
  11.         go wait.Until(func() {
  12.             err := http.ListenAndServe(s.Config.HealthzBindAddress+":"+strconv.Itoa(int(s.Config.HealthzPort)), nil)
  13.             if err != nil {
  14.                 glog.Errorf("Starting health server failed: %v", err)
  15.             }
  16.         }, 5*time.Second, wait.NeverStop)
  17.     }
  19.     ...
  21.     // Just loop forever for now...
  22.     s.Proxier.SyncLoop()
  23.     return nil
  24. }

Run方法关键代码非常easy。就是运行相应proxier的SyncLoop()。我们还是以iptables mode为例，看看它是怎样实现SyncLoop()的：

  1. pkg/proxy/iptables/proxier.go:416
  2. // SyncLoop runs periodic work.  This is expected to run as a goroutine or as the main loop of the app.  It does not return.
  3. func (proxier *Proxier) SyncLoop() {
  4.     t := time.NewTicker(proxier.syncPeriod)
  5.     defer t.Stop()
  6.     for {
  7.         <-t.C
  8.         glog.V(6).Infof("Periodic sync")
  9.         proxier.Sync()
  10.     }
  11. }

SyncLoop中，通过设置定时器，默认每30s会运行一次proxier.Sync()，能够通过--iptables-sync-period改动默认时间。

那我们继续跟进Sync()的代码：

  1. pkg/proxy/iptables/proxier.go:409
  2. // Sync is called to immediately synchronize the proxier state to iptables
  3. func (proxier *Proxier) Sync() {
  4.     proxier.mu.Lock()
  5.     defer proxier.mu.Unlock()
  6.     proxier.syncProxyRules()
  7. }

可见，终于还是调用proxier.syncProxyRules()。

前一节中创建ProxyServer的分析也是一样。终于watch到service/endpoint有更新时。都会调用到proxier.syncProxyRules()。

那以下我们就来看看proxier.syncProxyRules()的代码。

proxier.syncProxyRules

以下的proxier.syncProxyRules代码是iptables mode相应的实现。userspace mode的代码我就不贴了。

  1. pkg/proxy/iptables/proxier.go:791
  2. // This is where all of the iptables-save/restore calls happen.
  3. // The only other iptables rules are those that are setup in iptablesInit()
  4. // assumes proxier.mu is held
  5. func (proxier *Proxier) syncProxyRules() {
  6.     if proxier.throttle != nil {
  7.         proxier.throttle.Accept()
  8.     }
  9.     start := time.Now()
  10.     defer func() {
  11.         glog.V(4).Infof("syncProxyRules took %v", time.Since(start))
  12.     }()
  13.     // don't sync rules till we've received services and endpoints
  14.     if !proxier.haveReceivedEndpointsUpdate || !proxier.haveReceivedServiceUpdate {
  15.         glog.V(2).Info("Not syncing iptables until Services and Endpoints have been received from master")
  16.         return
  17.     }
  18.     glog.V(3).Infof("Syncing iptables rules")
  20.     // Create and link the kube services chain.
  21.     {
  22.         tablesNeedServicesChain := []utiliptables.Table{utiliptables.TableFilter, utiliptables.TableNAT}
  23.         for _, table := range tablesNeedServicesChain {
  24.             if _, err := proxier.iptables.EnsureChain(table, kubeServicesChain); err != nil {
  25.                 glog.Errorf("Failed to ensure that %s chain %s exists: %v", table, kubeServicesChain, err)
  26.                 return
  27.             }
  28.         }
  30.         tableChainsNeedJumpServices := []struct {
  31.             table utiliptables.Table
  32.             chain utiliptables.Chain
  33.         }{
  34.             {utiliptables.TableFilter, utiliptables.ChainOutput},
  35.             {utiliptables.TableNAT, utiliptables.ChainOutput},
  36.             {utiliptables.TableNAT, utiliptables.ChainPrerouting},
  37.         }
  38.         comment := "kubernetes service portals"
  39.         args := []string{"-m", "comment", "--comment", comment, "-j", string(kubeServicesChain)}
  40.         for _, tc := range tableChainsNeedJumpServices {
  41.             if _, err := proxier.iptables.EnsureRule(utiliptables.Prepend, tc.table, tc.chain, args...); err != nil {
  42.                 glog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", tc.table, tc.chain, kubeServicesChain, err)
  43.                 return
  44.             }
  45.         }
  46.     }
  48.     // Create and link the kube postrouting chain.
  49.     {
  50.         if _, err := proxier.iptables.EnsureChain(utiliptables.TableNAT, kubePostroutingChain); err != nil {
  51.             glog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, kubePostroutingChain, err)
  52.             return
  53.         }
  55.         comment := "kubernetes postrouting rules"
  56.         args := []string{"-m", "comment", "--comment", comment, "-j", string(kubePostroutingChain)}
  57.         if _, err := proxier.iptables.EnsureRule(utiliptables.Prepend, utiliptables.TableNAT, utiliptables.ChainPostrouting, args...); err != nil {
  58.             glog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", utiliptables.TableNAT, utiliptables.ChainPostrouting, kubePostroutingChain, err)
  59.             return
  60.         }
  61.     }
  63.     // Get iptables-save output so we can check for existing chains and rules.
  64.     // This will be a map of chain name to chain with rules as stored in iptables-save/iptables-restore
  65.     existingFilterChains := make(map[utiliptables.Chain]string)
  66.     iptablesSaveRaw, err := proxier.iptables.Save(utiliptables.TableFilter)
  67.     if err != nil { // if we failed to get any rules
  68.         glog.Errorf("Failed to execute iptables-save, syncing all rules: %v", err)
  69.     } else { // otherwise parse the output
  70.         existingFilterChains = utiliptables.GetChainLines(utiliptables.TableFilter, iptablesSaveRaw)
  71.     }
  73.     existingNATChains := make(map[utiliptables.Chain]string)
  74.     iptablesSaveRaw, err = proxier.iptables.Save(utiliptables.TableNAT)
  75.     if err != nil { // if we failed to get any rules
  76.         glog.Errorf("Failed to execute iptables-save, syncing all rules: %v", err)
  77.     } else { // otherwise parse the output
  78.         existingNATChains = utiliptables.GetChainLines(utiliptables.TableNAT, iptablesSaveRaw)
  79.     }
  81.     filterChains := bytes.NewBuffer(nil)
  82.     filterRules := bytes.NewBuffer(nil)
  83.     natChains := bytes.NewBuffer(nil)
  84.     natRules := bytes.NewBuffer(nil)
  86.     // Write table headers.
  87.     writeLine(filterChains, "*filter")
  88.     writeLine(natChains, "*nat")
  90.     // Make sure we keep stats for the top-level chains, if they existed
  91.     // (which most should have because we created them above).
  92.     if chain, ok := existingFilterChains[kubeServicesChain]; ok {
  93.         writeLine(filterChains, chain)
  94.     } else {
  95.         writeLine(filterChains, utiliptables.MakeChainLine(kubeServicesChain))
  96.     }
  97.     if chain, ok := existingNATChains[kubeServicesChain]; ok {
  98.         writeLine(natChains, chain)
  99.     } else {
  100.         writeLine(natChains, utiliptables.MakeChainLine(kubeServicesChain))
  101.     }
  102.     if chain, ok := existingNATChains[kubeNodePortsChain]; ok {
  103.         writeLine(natChains, chain)
  104.     } else {
  105.         writeLine(natChains, utiliptables.MakeChainLine(kubeNodePortsChain))
  106.     }
  107.     if chain, ok := existingNATChains[kubePostroutingChain]; ok {
  108.         writeLine(natChains, chain)
  109.     } else {
  110.         writeLine(natChains, utiliptables.MakeChainLine(kubePostroutingChain))
  111.     }
  112.     if chain, ok := existingNATChains[KubeMarkMasqChain]; ok {
  113.         writeLine(natChains, chain)
  114.     } else {
  115.         writeLine(natChains, utiliptables.MakeChainLine(KubeMarkMasqChain))
  116.     }
  118.     // Install the kubernetes-specific postrouting rules. We use a whole chain for
  119.     // this so that it is easier to flush and change, for example if the mark
  120.     // value should ever change.
  121.     writeLine(natRules, []string{
  122.         "-A", string(kubePostroutingChain),
  123.         "-m", "comment", "--comment", `"kubernetes service traffic requiring SNAT"`,
  124.         "-m", "mark", "--mark", proxier.masqueradeMark,
  125.         "-j", "MASQUERADE",
  126.     }...)
  128.     // Install the kubernetes-specific masquerade mark rule. We use a whole chain for
  129.     // this so that it is easier to flush and change, for example if the mark
  130.     // value should ever change.
  131.     writeLine(natRules, []string{
  132.         "-A", string(KubeMarkMasqChain),
  133.         "-j", "MARK", "--set-xmark", proxier.masqueradeMark,
  134.     }...)
  136.     // Accumulate NAT chains to keep.
  137.     activeNATChains := map[utiliptables.Chain]bool{} // use a map as a set
  139.     // Accumulate the set of local ports that we will be holding open once this update is complete
  140.     replacementPortsMap := map[localPort]closeable{}
  142.     // Build rules for each service.
  143.     for svcName, svcInfo := range proxier.serviceMap {
  144.         protocol := strings.ToLower(string(svcInfo.protocol))
  146.         // Create the per-service chain, retaining counters if possible.
  147.         svcChain := servicePortChainName(svcName, protocol)
  148.         if chain, ok := existingNATChains[svcChain]; ok {
  149.             writeLine(natChains, chain)
  150.         } else {
  151.             writeLine(natChains, utiliptables.MakeChainLine(svcChain))
  152.         }
  153.         activeNATChains[svcChain] = true
  155.         svcXlbChain := serviceLBChainName(svcName, protocol)
  156.         if svcInfo.onlyNodeLocalEndpoints {
  157.             // Only for services with the externalTraffic annotation set to OnlyLocal
  158.             // create the per-service LB chain, retaining counters if possible.
  159.             if lbChain, ok := existingNATChains[svcXlbChain]; ok {
  160.                 writeLine(natChains, lbChain)
  161.             } else {
  162.                 writeLine(natChains, utiliptables.MakeChainLine(svcXlbChain))
  163.             }
  164.             activeNATChains[svcXlbChain] = true
  165.         } else if activeNATChains[svcXlbChain] {
  166.             // Cleanup the previously created XLB chain for this service
  167.             delete(activeNATChains, svcXlbChain)
  168.         }
  170.         // Capture the clusterIP.
  171.         args := []string{
  172.             "-A", string(kubeServicesChain),
  173.             "-m", "comment", "--comment", fmt.Sprintf(`"%s cluster IP"`, svcName.String()),
  174.             "-m", protocol, "-p", protocol,
  175.             "-d", fmt.Sprintf("%s/32", svcInfo.clusterIP.String()),
  176.             "--dport", fmt.Sprintf("%d", svcInfo.port),
  177.         }
  178.         if proxier.masqueradeAll {
  179.             writeLine(natRules, append(args, "-j", string(KubeMarkMasqChain))...)
  180.         }
  181.         if len(proxier.clusterCIDR) > 0 {
  182.             writeLine(natRules, append(args, "! -s", proxier.clusterCIDR, "-j", string(KubeMarkMasqChain))...)
  183.         }
  184.         writeLine(natRules, append(args, "-j", string(svcChain))...)
  186.         // Capture externalIPs.
  187.         for _, externalIP := range svcInfo.externalIPs {
  188.             // If the "external" IP happens to be an IP that is local to this
  189.             // machine, hold the local port open so no other process can open it
  190.             // (because the socket might open but it would never work).
  191.             if local, err := isLocalIP(externalIP); err != nil {
  192.                 glog.Errorf("can't determine if IP is local, assuming not: %v", err)
  193.             } else if local {
  194.                 lp := localPort{
  195.                     desc:     "externalIP for " + svcName.String(),
  196.                     ip:       externalIP,
  197.                     port:     svcInfo.port,
  198.                     protocol: protocol,
  199.                 }
  200.                 if proxier.portsMap[lp] != nil {
  201.                     glog.V(4).Infof("Port %s was open before and is still needed", lp.String())
  202.                     replacementPortsMap[lp] = proxier.portsMap[lp]
  203.                 } else {
  204.                     socket, err := proxier.portMapper.OpenLocalPort(&lp)
  205.                     if err != nil {
  206.                         glog.Errorf("can't open %s, skipping this externalIP: %v", lp.String(), err)
  207.                         continue
  208.                     }
  209.                     replacementPortsMap[lp] = socket
  210.                 }
  211.             } // We're holding the port, so it's OK to install iptables rules.
  212.             args := []string{
  213.                 "-A", string(kubeServicesChain),
  214.                 "-m", "comment", "--comment", fmt.Sprintf(`"%s external IP"`, svcName.String()),
  215.                 "-m", protocol, "-p", protocol,
  216.                 "-d", fmt.Sprintf("%s/32", externalIP),
  217.                 "--dport", fmt.Sprintf("%d", svcInfo.port),
  218.             }
  219.             // We have to SNAT packets to external IPs.
  220.             writeLine(natRules, append(args, "-j", string(KubeMarkMasqChain))...)
  222.             // Allow traffic for external IPs that does not come from a bridge (i.e. not from a container)
  223.             // nor from a local process to be forwarded to the service.
  224.             // This rule roughly translates to "all traffic from off-machine".
  225.             // This is imperfect in the face of network plugins that might not use a bridge, but we can revisit that later.
  226.             externalTrafficOnlyArgs := append(args,
  227.                 "-m", "physdev", "!", "--physdev-is-in",
  228.                 "-m", "addrtype", "!", "--src-type", "LOCAL")
  229.             writeLine(natRules, append(externalTrafficOnlyArgs, "-j", string(svcChain))...)
  230.             dstLocalOnlyArgs := append(args, "-m", "addrtype", "--dst-type", "LOCAL")
  231.             // Allow traffic bound for external IPs that happen to be recognized as local IPs to stay local.
  232.             // This covers cases like GCE load-balancers which get added to the local routing table.
  233.             writeLine(natRules, append(dstLocalOnlyArgs, "-j", string(svcChain))...)
  234.         }
  236.         // Capture load-balancer ingress.
  237.         for _, ingress := range svcInfo.loadBalancerStatus.Ingress {
  238.             if ingress.IP != "" {
  239.                 // create service firewall chain
  240.                 fwChain := serviceFirewallChainName(svcName, protocol)
  241.                 if chain, ok := existingNATChains[fwChain]; ok {
  242.                     writeLine(natChains, chain)
  243.                 } else {
  244.                     writeLine(natChains, utiliptables.MakeChainLine(fwChain))
  245.                 }
  246.                 activeNATChains[fwChain] = true
  247.                 // The service firewall rules are created based on ServiceSpec.loadBalancerSourceRanges field.
  248.                 // This currently works for loadbalancers that preserves source ips.
  249.                 // For loadbalancers which direct traffic to service NodePort, the firewall rules will not apply.
  251.                 args := []string{
  252.                     "-A", string(kubeServicesChain),
  253.                     "-m", "comment", "--comment", fmt.Sprintf(`"%s loadbalancer IP"`, svcName.String()),
  254.                     "-m", protocol, "-p", protocol,
  255.                     "-d", fmt.Sprintf("%s/32", ingress.IP),
  256.                     "--dport", fmt.Sprintf("%d", svcInfo.port),
  257.                 }
  258.                 // jump to service firewall chain
  259.                 writeLine(natRules, append(args, "-j", string(fwChain))...)
  261.                 args = []string{
  262.                     "-A", string(fwChain),
  263.                     "-m", "comment", "--comment", fmt.Sprintf(`"%s loadbalancer IP"`, svcName.String()),
  264.                 }
  266.                 // Each source match rule in the FW chain may jump to either the SVC or the XLB chain
  267.                 chosenChain := svcXlbChain
  268.                 // If we are proxying globally, we need to masquerade in case we cross nodes.
  269.                 // If we are proxying only locally, we can retain the source IP.
  270.                 if !svcInfo.onlyNodeLocalEndpoints {
  271.                     writeLine(natRules, append(args, "-j", string(KubeMarkMasqChain))...)
  272.                     chosenChain = svcChain
  273.                 }
  275.                 if len(svcInfo.loadBalancerSourceRanges) == 0 {
  276.                     // allow all sources, so jump directly to the KUBE-SVC or KUBE-XLB chain
  277.                     writeLine(natRules, append(args, "-j", string(chosenChain))...)
  278.                 } else {
  279.                     // firewall filter based on each source range
  280.                     allowFromNode := false
  281.                     for _, src := range svcInfo.loadBalancerSourceRanges {
  282.                         writeLine(natRules, append(args, "-s", src, "-j", string(chosenChain))...)
  283.                         // ignore error because it has been validated
  284.                         _, cidr, _ := net.ParseCIDR(src)
  285.                         if cidr.Contains(proxier.nodeIP) {
  286.                             allowFromNode = true
  287.                         }
  288.                     }
  289.                     // generally, ip route rule was added to intercept request to loadbalancer vip from the
  290.                     // loadbalancer's backend hosts. In this case, request will not hit the loadbalancer but loop back directly.
  291.                     // Need to add the following rule to allow request on host.
  292.                     if allowFromNode {
  293.                         writeLine(natRules, append(args, "-s", fmt.Sprintf("%s/32", ingress.IP), "-j", string(chosenChain))...)
  294.                     }
  295.                 }
  297.                 // If the packet was able to reach the end of firewall chain, then it did not get DNATed.
  298.                 // It means the packet cannot go thru the firewall, then mark it for DROP
  299.                 writeLine(natRules, append(args, "-j", string(KubeMarkDropChain))...)
  300.             }
  301.         }
  303.         // Capture nodeports.  If we had more than 2 rules it might be
  304.         // worthwhile to make a new per-service chain for nodeport rules, but
  305.         // with just 2 rules it ends up being a waste and a cognitive burden.
  306.         if svcInfo.nodePort != 0 {
  307.             // Hold the local port open so no other process can open it
  308.             // (because the socket might open but it would never work).
  309.             lp := localPort{
  310.                 desc:     "nodePort for " + svcName.String(),
  311.                 ip:       "",
  312.                 port:     svcInfo.nodePort,
  313.                 protocol: protocol,
  314.             }
  315.             if proxier.portsMap[lp] != nil {
  316.                 glog.V(4).Infof("Port %s was open before and is still needed", lp.String())
  317.                 replacementPortsMap[lp] = proxier.portsMap[lp]
  318.             } else {
  319.                 socket, err := proxier.portMapper.OpenLocalPort(&lp)
  320.                 if err != nil {
  321.                     glog.Errorf("can't open %s, skipping this nodePort: %v", lp.String(), err)
  322.                     continue
  323.                 }
  324.                 if lp.protocol == "udp" {
  325.                     proxier.clearUdpConntrackForPort(lp.port)
  326.                 }
  327.                 replacementPortsMap[lp] = socket
  328.             } // We're holding the port, so it's OK to install iptables rules.
  330.             args := []string{
  331.                 "-A", string(kubeNodePortsChain),
  332.                 "-m", "comment", "--comment", svcName.String(),
  333.                 "-m", protocol, "-p", protocol,
  334.                 "--dport", fmt.Sprintf("%d", svcInfo.nodePort),
  335.             }
  336.             if !svcInfo.onlyNodeLocalEndpoints {
  337.                 // Nodeports need SNAT, unless they're local.
  338.                 writeLine(natRules, append(args, "-j", string(KubeMarkMasqChain))...)
  339.                 // Jump to the service chain.
  340.                 writeLine(natRules, append(args, "-j", string(svcChain))...)
  341.             } else {
  342.                 // TODO: Make all nodePorts jump to the firewall chain.
  343.                 // Currently we only create it for loadbalancers (#33586).
  344.                 writeLine(natRules, append(args, "-j", string(svcXlbChain))...)
  345.             }
  346.         }
  348.         // If the service has no endpoints then reject packets.
  349.         if len(proxier.endpointsMap[svcName]) == 0 {
  350.             writeLine(filterRules,
  351.                 "-A", string(kubeServicesChain),
  352.                 "-m", "comment", "--comment", fmt.Sprintf(`"%s has no endpoints"`, svcName.String()),
  353.                 "-m", protocol, "-p", protocol,
  354.                 "-d", fmt.Sprintf("%s/32", svcInfo.clusterIP.String()),
  355.                 "--dport", fmt.Sprintf("%d", svcInfo.port),
  356.                 "-j", "REJECT",
  357.             )
  358.             continue
  359.         }
  361.         // Generate the per-endpoint chains.  We do this in multiple passes so we
  362.         // can group rules together.
  363.         // These two slices parallel each other - keep in sync
  364.         endpoints := make([]*endpointsInfo, 0)
  365.         endpointChains := make([]utiliptables.Chain, 0)
  366.         for _, ep := range proxier.endpointsMap[svcName] {
  367.             endpoints = append(endpoints, ep)
  368.             endpointChain := servicePortEndpointChainName(svcName, protocol, ep.ip)
  369.             endpointChains = append(endpointChains, endpointChain)
  371.             // Create the endpoint chain, retaining counters if possible.
  372.             if chain, ok := existingNATChains[utiliptables.Chain(endpointChain)]; ok {
  373.                 writeLine(natChains, chain)
  374.             } else {
  375.                 writeLine(natChains, utiliptables.MakeChainLine(endpointChain))
  376.             }
  377.             activeNATChains[endpointChain] = true
  378.         }
  380.         // First write session affinity rules, if applicable.
  381.         if svcInfo.sessionAffinityType == api.ServiceAffinityClientIP {
  382.             for _, endpointChain := range endpointChains {
  383.                 writeLine(natRules,
  384.                     "-A", string(svcChain),
  385.                     "-m", "comment", "--comment", svcName.String(),
  386.                     "-m", "recent", "--name", string(endpointChain),
  387.                     "--rcheck", "--seconds", fmt.Sprintf("%d", svcInfo.stickyMaxAgeMinutes*60), "--reap",
  388.                     "-j", string(endpointChain))
  389.             }
  390.         }
  392.         // Now write loadbalancing & DNAT rules.
  393.         n := len(endpointChains)
  394.         for i, endpointChain := range endpointChains {
  395.             // Balancing rules in the per-service chain.
  396.             args := []string{
  397.                 "-A", string(svcChain),
  398.                 "-m", "comment", "--comment", svcName.String(),
  399.             }
  400.             if i < (n - 1) {
  401.                 // Each rule is a probabilistic match.
  402.                 args = append(args,
  403.                     "-m", "statistic",
  404.                     "--mode", "random",
  405.                     "--probability", fmt.Sprintf("%0.5f", 1.0/float64(n-i)))
  406.             }
  407.             // The final (or only if n == 1) rule is a guaranteed match.
  408.             args = append(args, "-j", string(endpointChain))
  409.             writeLine(natRules, args...)
  411.             // Rules in the per-endpoint chain.
  412.             args = []string{
  413.                 "-A", string(endpointChain),
  414.                 "-m", "comment", "--comment", svcName.String(),
  415.             }
  416.             // Handle traffic that loops back to the originator with SNAT.
  417.             writeLine(natRules, append(args,
  418.                 "-s", fmt.Sprintf("%s/32", strings.Split(endpoints[i].ip, ":")[0]),
  419.                 "-j", string(KubeMarkMasqChain))...)
  420.             // Update client-affinity lists.
  421.             if svcInfo.sessionAffinityType == api.ServiceAffinityClientIP {
  422.                 args = append(args, "-m", "recent", "--name", string(endpointChain), "--set")
  423.             }
  424.             // DNAT to final destination.
  425.             args = append(args, "-m", protocol, "-p", protocol, "-j", "DNAT", "--to-destination", endpoints[i].ip)
  426.             writeLine(natRules, args...)
  427.         }
  429.         // The logic below this applies only if this service is marked as OnlyLocal
  430.         if !svcInfo.onlyNodeLocalEndpoints {
  431.             continue
  432.         }
  434.         // Now write ingress loadbalancing & DNAT rules only for services that have a localOnly annotation
  435.         // TODO - This logic may be combinable with the block above that creates the svc balancer chain
  436.         localEndpoints := make([]*endpointsInfo, 0)
  437.         localEndpointChains := make([]utiliptables.Chain, 0)
  438.         for i := range endpointChains {
  439.             if endpoints[i].localEndpoint {
  440.                 // These slices parallel each other; must be kept in sync
  441.                 localEndpoints = append(localEndpoints, endpoints[i])
  442.                 localEndpointChains = append(localEndpointChains, endpointChains[i])
  443.             }
  444.         }
  445.         // First rule in the chain redirects all pod -> external vip traffic to the
  446.         // Service's ClusterIP instead. This happens whether or not we have local
  447.         // endpoints; only if clusterCIDR is specified
  448.         if len(proxier.clusterCIDR) > 0 {
  449.             args = []string{
  450.                 "-A", string(svcXlbChain),
  451.                 "-m", "comment", "--comment",
  452.                 fmt.Sprintf(`"Redirect pods trying to reach external loadbalancer VIP to clusterIP"`),
  453.                 "-s", proxier.clusterCIDR,
  454.                 "-j", string(svcChain),
  455.             }
  456.             writeLine(natRules, args...)
  457.         }
  459.         numLocalEndpoints := len(localEndpointChains)
  460.         if numLocalEndpoints == 0 {
  461.             // Blackhole all traffic since there are no local endpoints
  462.             args := []string{
  463.                 "-A", string(svcXlbChain),
  464.                 "-m", "comment", "--comment",
  465.                 fmt.Sprintf(`"%s has no local endpoints"`, svcName.String()),
  466.                 "-j",
  467.                 string(KubeMarkDropChain),
  468.             }
  469.             writeLine(natRules, args...)
  470.         } else {
  471.             // Setup probability filter rules only over local endpoints
  472.             for i, endpointChain := range localEndpointChains {
  473.                 // Balancing rules in the per-service chain.
  474.                 args := []string{
  475.                     "-A", string(svcXlbChain),
  476.                     "-m", "comment", "--comment",
  477.                     fmt.Sprintf(`"Balancing rule %d for %s"`, i, svcName.String()),
  478.                 }
  479.                 if i < (numLocalEndpoints - 1) {
  480.                     // Each rule is a probabilistic match.
  481.                     args = append(args,
  482.                         "-m", "statistic",
  483.                         "--mode", "random",
  484.                         "--probability", fmt.Sprintf("%0.5f", 1.0/float64(numLocalEndpoints-i)))
  485.                 }
  486.                 // The final (or only if n == 1) rule is a guaranteed match.
  487.                 args = append(args, "-j", string(endpointChain))
  488.                 writeLine(natRules, args...)
  489.             }
  490.         }
  491.     }
  493.     // Delete chains no longer in use.
  494.     for chain := range existingNATChains {
  495.         if !activeNATChains[chain] {
  496.             chainString := string(chain)
  497.             if !strings.HasPrefix(chainString, "KUBE-SVC-") && !strings.HasPrefix(chainString, "KUBE-SEP-") && !strings.HasPrefix(chainString, "KUBE-FW-") && !strings.HasPrefix(chainString, "KUBE-XLB-") {
  498.                 // Ignore chains that aren't ours.
  499.                 continue
  500.             }
  501.             // We must (as per iptables) write a chain-line for it, which has
  502.             // the nice effect of flushing the chain.  Then we can remove the
  503.             // chain.
  504.             writeLine(natChains, existingNATChains[chain])
  505.             writeLine(natRules, "-X", chainString)
  506.         }
  507.     }
  509.     // Finally, tail-call to the nodeports chain.  This needs to be after all
  510.     // other service portal rules.
  511.     writeLine(natRules,
  512.         "-A", string(kubeServicesChain),
  513.         "-m", "comment", "--comment", `"kubernetes service nodeports; NOTE: this must be the last rule in this chain"`,
  514.         "-m", "addrtype", "--dst-type", "LOCAL",
  515.         "-j", string(kubeNodePortsChain))
  517.     // Write the end-of-table markers.
  518.     writeLine(filterRules, "COMMIT")
  519.     writeLine(natRules, "COMMIT")
  521.     // Sync rules.
  522.     // NOTE: NoFlushTables is used so we don't flush non-kubernetes chains in the table.
  523.     filterLines := append(filterChains.Bytes(), filterRules.Bytes()...)
  524.     natLines := append(natChains.Bytes(), natRules.Bytes()...)
  525.     lines := append(filterLines, natLines...)
  527.     glog.V(3).Infof("Restoring iptables rules: %s", lines)
  528.     err = proxier.iptables.RestoreAll(lines, utiliptables.NoFlushTables, utiliptables.RestoreCounters)
  529.     if err != nil {
  530.         glog.Errorf("Failed to execute iptables-restore: %v\nRules:\n%s", err, lines)
  531.         // Revert new local ports.
  532.         revertPorts(replacementPortsMap, proxier.portsMap)
  533.         return
  534.     }
  536.     // Close old local ports and save new ones.
  537.     for k, v := range proxier.portsMap {
  538.         if replacementPortsMap[k] == nil {
  539.             v.Close()
  540.         }
  541.     }
  542.     proxier.portsMap = replacementPortsMap
  543. }

看到这么长的方法，本来想多写一点分析凝视的，结果我看完已经肌无力了。

假设你自己又k8s的环境。找一台node，查看其iptables，对着以下的代码来看会好非常多。

假设你没有环境，没关系，能够參考到我的上一篇博文kube-proxy工作原理查看相应的Example。

总结

  • kube-proxy实现了两种linux下的proxy mode：userspace和iptables，实现了一种windows下的proxy mode：userspace。

  • kube-proxy通过周期性的List and Watch kube-apiserver的all service and endpiont Resources。通过Channels传给相应的Broadcaster，由Broadcaster Notify给Proxier注冊的Listener。List周期默认15min，可通过--config-sync-period配置。
  • Listener实现OnServiceUpdate和OnEndpointsUpdate接口，终于调用proxier.syncProxyRules()更新iptables。
  • 另外，Proxy Run方法负责周期性的调用proxier.syncProxyRules()更新iptables，默认30s一次，可通过--iptables-sync-period配置。

kube-proxy源代码分析的更多相关文章

  1. Proxy源代码分析——谈谈如何学习Linux网络编程

    Linux是一个可靠性非常高的操作系统,但是所有用过Linux的朋友都会感觉到, Linux和Windows这样的"傻瓜"操作系统(这里丝毫没有贬低Windows的意思,相反这应该 ...

  2. Proxy源代码分析--谈谈如何学习Linux网络编程

    http://blog.csdn.net/cloudtech/article/details/1823531 Linux是一个可靠性非常高的操作系统,但是所有用过Linux的朋友都会感觉到,Linux ...

  3. Hadoop源代码分析

    http://wenku.baidu.com/link?url=R-QoZXhc918qoO0BX6eXI9_uPU75whF62vFFUBIR-7c5XAYUVxDRX5Rs6QZR9hrBnUdM ...

  4. mybatis源代码分析：mybatis延迟加载机制改进

    在上一篇博客<mybatis源代码分析:深入了解mybatis延迟加载机制>讲诉了mybatis延迟加载的具体机制及实现原理. 可以看出,如果查询结果对象中有一个属性是需要延迟加载的,那整 ...

  5. Android系统进程间通信Binder机制在应用程序框架层的Java接口源代码分析

    文章转载至CSDN社区罗升阳的安卓之旅,原文地址:http://blog.csdn.net/luoshengyang/article/details/6642463 在前面几篇文章中,我们详细介绍了A ...

  6. Android系统进程间通信（IPC）机制Binder中的Client获得Server远程接口过程源代码分析

    文章转载至CSDN社区罗升阳的安卓之旅,原文地址:http://blog.csdn.net/luoshengyang/article/details/6633311 在上一篇文章中,我 们分析了And ...

  7. Android应用Activity、Dialog、PopWindow、Toast窗体加入机制及源代码分析

    [工匠若水 http://blog.csdn.net/yanbober 转载烦请注明出处.尊重劳动成果] 1 背景 之所以写这一篇博客的原因是由于之前有写过一篇<Android应用setCont ...

  8. struts2源代码分析(个人觉得非常经典)

    读者如果曾经学习过Struts1.x或者有过Struts1.x的开发经验,那么千万不要想当然地以为这一章可以跳过.实际上Struts1.x与Struts2并无我们想象的血缘关系.虽然Struts2的开 ...

  9. Android HttpURLConnection源代码分析

    Android HttpURLConnection源代码分析 之前写过HttpURLConnection与HttpClient的差别及选择.后来又分析了Volley的源代码. 近期又遇到了问题,想在V ...

随机推荐

  1. mysql error You must reset your password using ALTER USER statement before executing this statement.

    mysql修改密码Your password does not satisfy the current policy requirements 出现这个问题的原因是:密码过于简单.刚安装的mysql的 ...

  2. localhost，127.0.0.1 和 本机IP 三者的区别

    localhost.127.0.0.1和本机IP的区别如下: 1.首先 localhost 是一个域名,在过去它指向 127.0.0.1 这个IP地址.在操作系统支持 ipv6 后,它同时还指向ipv ...

  3. Linux下怎么确定Nginx安装目录

    linux环境下,怎么确定nginx是以那个config文件启动的? 输入命令行: ps  -ef | grep nginx 摁回车,将出现如下图片: master process 后面的就是 ngi ...

  4. Python数据分析笔记

    最近在看Python数据分析这本书,随手记录一下读书笔记. 工作环境 本书中推荐了edm和ipython作为数据分析的环境,我还是刚开始使用这种集成的环境,觉得交互方面,比传统的命令行方式提高了不少. ...

  5. 在ubuntu中搜索文件或文件夹的方法

    版权声明:本文为博主原创文章,转载请注明出处. https://blog.csdn.net/dcrmg/article/details/78000961 1. whereis+文件名 用于程序名的搜索 ...

  6. .AVLFile Extension

    .AVLFile Extension File Type 1ArcView Legend File   Developer ESRI Popularity           4.1 (7 Votes ...

  7. [Link]Hive资料整理

    Hive SQL的编译过程 Hive学习分享 IBM Hive

  8. jquery fullPage

    FROM : http://www.dowebok.com/77.html 应用: http://txhd.163.com/

  9. ASP.NET C#根据HTML页面导出PDF

    在启明星采购系统里,新增了导出PDF功能.整个功能使用了第三方软件 wkhtmltopdf(下载) 官网 https://wkhtmltopdf.org/ 提供有更多版本下载 他可以把HTML页面转换 ...

  10. java操作mongodb（连接池）（转）

    原文链接: java操作mongodb(连接池) Mongo的实例其实就是一个数据库连接池,这个连接池里默认有10个链接.我们没有必要重新实现这个链接池,但是我们可以更改这个连接池的配置.因为Mong ...