Skip to content

Instantly share code, notes, and snippets.

@quater

quater/kops_rolling_update_when_autospotting_enabled

Created September 3, 2018 14:45
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save quater/79a9259b06e106afd31e05706d9d6759 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save quater/79a9259b06e106afd31e05706d9d6759 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
kops_rolling_update_when_autospotting_enabled
# Command
$ kops rolling-update cluster shine.dev.example.org --state s3://example-org-cluster-config --v 10
# Output
I0903 15:12:55.477031 15573 factory.go:68] state store s3://example-org-cluster-config
I0903 15:12:55.930741 15573 s3context.go:198] Checking default bucket encryption "example-org-cluster-config"
I0903 15:12:55.930762 15573 s3context.go:203] Calling S3 GetBucketEncryption Bucket="example-org-cluster-config"
I0903 15:12:56.321236 15573 s3context.go:210] Unable to read bucket encryption policy: will encrypt using AES256
I0903 15:12:56.321253 15573 s3context.go:182] Found bucket "example-org-cluster-config" in region "us-east-1" with default encryption set to false
I0903 15:12:56.321269 15573 s3fs.go:219] Reading file "s3://example-org-cluster-config/shine.dev.example.org/config"
I0903 15:12:56.506045 15573 loader.go:357] Config loaded from file /home/quatmin/.kube/config
I0903 15:12:56.507101 15573 round_trippers.go:386] curl -k -v -XGET -H "Accept: application/json, */*" -H "User-Agent: kops/v0.0.0 (linux/amd64) kubernetes/$Format" -H "Authorization: Basic YWRtaW46SomeKeyyyyyyyyyyyyyyyyyyyyyyyyyyyyJhcFh0b0g=" https://api.shine.dev.example.org/api/v1/nodes
I0903 15:12:56.882817 15573 round_trippers.go:405] GET https://api.shine.dev.example.org/api/v1/nodes 200 OK in 375 milliseconds
I0903 15:12:56.882835 15573 round_trippers.go:411] Response Headers:
I0903 15:12:56.882845 15573 round_trippers.go:414] Content-Type: application/json
I0903 15:12:56.882854 15573 round_trippers.go:414] Date: Mon, 03 Sep 2018 14:12:56 GMT
I0903 15:12:56.968883 15573 request.go:874] Response Body:
{
"apiVersion": "v1",
"items": [
{
"metadata": {
"annotations": {
"node.alpha.kubernetes.io/ttl": "0",
"volumes.kubernetes.io/controller-managed-attach-detach": "true"
},
"creationTimestamp": "2018-09-03T13:55:37Z",
"labels": {
"beta.kubernetes.io/arch": "amd64",
"beta.kubernetes.io/instance-type": "t2.medium",
"beta.kubernetes.io/os": "linux",
"failure-domain.beta.kubernetes.io/region": "us-east-1",
"failure-domain.beta.kubernetes.io/zone": "us-east-1a",
"kops.k8s.io/instancegroup": "master-us-east-1a",
"kubernetes.io/hostname": "ip-172-20-38-150.ec2.internal",
"kubernetes.io/role": "master",
"node-role.kubernetes.io/master": ""
},
"name": "ip-172-20-38-150.ec2.internal",
"resourceVersion": "25820",
"selfLink": "/api/v1/nodes/ip-172-20-38-150.ec2.internal",
"uid": "09580942-af81-11e8-9a9b-12064eb09e52"
},
"spec": {
"externalID": "i-05561d9f029cb3768",
"podCIDR": "100.96.0.0/24",
"providerID": "aws:///us-east-1a/i-05561d9f029cb3768",
"taints": [
{
"effect": "NoSchedule",
"key": "node-role.kubernetes.io/master"
}
]
},
"status": {
"addresses": [
{
"address": "172.20.38.150",
"type": "InternalIP"
},
{
"address": "ip-172-20-38-150.ec2.internal",
"type": "InternalDNS"
},
{
"address": "ip-172-20-38-150.ec2.internal",
"type": "Hostname"
}
],
"allocatable": {
"cpu": "2",
"ephemeral-storage": "57856285806",
"hugepages-2Mi": "0",
"memory": "3946708Ki",
"pods": "110"
},
"capacity": {
"cpu": "2",
"ephemeral-storage": "62778088Ki",
"hugepages-2Mi": "0",
"memory": "4049108Ki",
"pods": "110"
},
"conditions": [
{
"lastHeartbeatTime": "2018-09-03T14:12:50Z",
"lastTransitionTime": "2018-09-03T13:55:37Z",
"message": "kubelet has sufficient disk space available",
"reason": "KubeletHasSufficientDisk",
"status": "False",
"type": "OutOfDisk"
},
{
"lastHeartbeatTime": "2018-09-03T14:12:50Z",
"lastTransitionTime": "2018-09-03T13:55:37Z",
"message": "kubelet has sufficient memory available",
"reason": "KubeletHasSufficientMemory",
"status": "False",
"type": "MemoryPressure"
},
{
"lastHeartbeatTime": "2018-09-03T14:12:50Z",
"lastTransitionTime": "2018-09-03T13:55:37Z",
"message": "kubelet has no disk pressure",
"reason": "KubeletHasNoDiskPressure",
"status": "False",
"type": "DiskPressure"
},
{
"lastHeartbeatTime": "2018-09-03T14:12:50Z",
"lastTransitionTime": "2018-09-03T13:55:37Z",
"message": "kubelet has sufficient PID available",
"reason": "KubeletHasSufficientPID",
"status": "False",
"type": "PIDPressure"
},
{
"lastHeartbeatTime": "2018-09-03T14:12:50Z",
"lastTransitionTime": "2018-09-03T13:56:17Z",
"message": "kubelet is posting ready status",
"reason": "KubeletReady",
"status": "True",
"type": "Ready"
}
],
"daemonEndpoints": {
"kubeletEndpoint": {
"Port": 10250
}
},
"images": [
{
"names": [
"quay.io/calico/node@sha256:7758c25549fcfe677699bbcd3c279b3a174e7cbbbf9d16f3d71713d68f695dfb",
"quay.io/calico/node:v2.6.7"
],
"sizeBytes": 281572001
},
{
"names": [
"protokube:1.10.0-beta.1"
],
"sizeBytes": 277860377
},
{
"names": [
"k8s.gcr.io/kube-apiserver@sha256:ca54685b89b3e1809ea3fa9936e32e3a05083a84483813604178275e02352454",
"k8s.gcr.io/kube-apiserver:v1.10.5"
],
"sizeBytes": 228059419
},
{
"names": [
"k8s.gcr.io/kube-controller-manager@sha256:20afa70465a92fd3d573ddc78351fb1d69415e65e7e8e957adae19d60d75960d",
"k8s.gcr.io/kube-controller-manager:v1.10.5"
],
"sizeBytes": 150807988
},
{
"names": [
"kope/dns-controller@sha256:a4f5c5490b23e6bad32028d223b172f3fc9a6d4251cc5eea6e5e389c8dc12024",
"kope/dns-controller:1.10.0"
],
"sizeBytes": 119435348
},
{
"names": [
"k8s.gcr.io/kube-proxy@sha256:b81228e8ad694f05a5a6e035167ad705600aead5cfd63628a38984fc60f0b989",
"k8s.gcr.io/kube-proxy:v1.10.5"
],
"sizeBytes": 97861102
},
{
"names": [
"quay.io/calico/cni@sha256:3a23e093b1e98cf232a226fedff591d33919f5297f016a41d8012efc83b23a84",
"quay.io/calico/cni:v1.11.2"
],
"sizeBytes": 70781000
},
{
"names": [
"k8s.gcr.io/kube-scheduler@sha256:ccd7da1c35fefdb8077f80baf0724b861b94b3fc182ae0b5e0b7644257a0dd41",
"k8s.gcr.io/kube-scheduler:v1.10.5"
],
"sizeBytes": 51195168
},
{
"names": [
"k8s.gcr.io/coredns@sha256:db2bf53126ed1c761d5a41f24a1b82a461c85f736ff6e90542e9522be4757848",
"k8s.gcr.io/coredns:1.1.3"
],
"sizeBytes": 45564078
},
{
"names": [
"k8s.gcr.io/etcd@sha256:19544a655157fb089b62d4dac02bbd095f82ca245dd5e31dd1684d175b109947",
"k8s.gcr.io/etcd:2.2.1"
],
"sizeBytes": 28191895
},
{
"names": [
"k8s.gcr.io/pause-amd64@sha256:163ac025575b775d1c0f9bf0bdd0f086883171eb475b5068e7defa4ca9e76516",
"k8s.gcr.io/pause-amd64:3.0"
],
"sizeBytes": 746888
}
],
"nodeInfo": {
"architecture": "amd64",
"bootID": "d51dcfc6-cb19-47f9-bdbe-c591fa41f759",
"containerRuntimeVersion": "docker://17.3.2",
"kernelVersion": "4.9.0-6-amd64",
"kubeProxyVersion": "v1.10.5",
"kubeletVersion": "v1.10.5",
"machineID": "84341936367e42d1bc4a506326826190",
"operatingSystem": "linux",
"osImage": "Debian GNU/Linux 9 (stretch)",
"systemUUID": "EC2F12E5-9678-F308-D2F3-615AC07F7CBE"
}
}
},
{
"metadata": {
"annotations": {
"node.alpha.kubernetes.io/ttl": "0",
"volumes.kubernetes.io/controller-managed-attach-detach": "true"
},
"creationTimestamp": "2018-09-03T13:58:08Z",
"labels": {
"beta.kubernetes.io/arch": "amd64",
"beta.kubernetes.io/instance-type": "t2.medium",
"beta.kubernetes.io/os": "linux",
"failure-domain.beta.kubernetes.io/region": "us-east-1",
"failure-domain.beta.kubernetes.io/zone": "us-east-1a",
"kops.k8s.io/instancegroup": "nodes",
"kubernetes.io/hostname": "ip-172-20-44-26.ec2.internal",
"kubernetes.io/role": "node",
"node-role.kubernetes.io/node": ""
},
"name": "ip-172-20-44-26.ec2.internal",
"resourceVersion": "25821",
"selfLink": "/api/v1/nodes/ip-172-20-44-26.ec2.internal",
"uid": "637b474e-af81-11e8-9a9b-12064eb09e52"
},
"spec": {
"externalID": "i-0bde1609596da26fa",
"podCIDR": "100.96.1.0/24",
"providerID": "aws:///us-east-1a/i-0bde1609596da26fa"
},
"status": {
"addresses": [
{
"address": "172.20.44.26",
"type": "InternalIP"
},
{
"address": "ip-172-20-44-26.ec2.internal",
"type": "InternalDNS"
},
{
"address": "ip-172-20-44-26.ec2.internal",
"type": "Hostname"
}
],
"allocatable": {
"cpu": "2",
"ephemeral-storage": "115834311284",
"hugepages-2Mi": "0",
"memory": "3946712Ki",
"pods": "110"
},
"capacity": {
"cpu": "2",
"ephemeral-storage": "125688272Ki",
"hugepages-2Mi": "0",
"memory": "4049112Ki",
"pods": "110"
},
"conditions": [
{
"lastHeartbeatTime": "2018-09-03T14:12:50Z",
"lastTransitionTime": "2018-09-03T13:58:08Z",
"message": "kubelet has sufficient disk space available",
"reason": "KubeletHasSufficientDisk",
"status": "False",
"type": "OutOfDisk"
},
{
"lastHeartbeatTime": "2018-09-03T14:12:50Z",
"lastTransitionTime": "2018-09-03T13:58:08Z",
"message": "kubelet has sufficient memory available",
"reason": "KubeletHasSufficientMemory",
"status": "False",
"type": "MemoryPressure"
},
{
"lastHeartbeatTime": "2018-09-03T14:12:50Z",
"lastTransitionTime": "2018-09-03T13:58:08Z",
"message": "kubelet has no disk pressure",
"reason": "KubeletHasNoDiskPressure",
"status": "False",
"type": "DiskPressure"
},
{
"lastHeartbeatTime": "2018-09-03T14:12:50Z",
"lastTransitionTime": "2018-09-03T13:58:08Z",
"message": "kubelet has sufficient PID available",
"reason": "KubeletHasSufficientPID",
"status": "False",
"type": "PIDPressure"
},
{
"lastHeartbeatTime": "2018-09-03T14:12:50Z",
"lastTransitionTime": "2018-09-03T13:58:39Z",
"message": "kubelet is posting ready status",
"reason": "KubeletReady",
"status": "True",
"type": "Ready"
}
],
"daemonEndpoints": {
"kubeletEndpoint": {
"Port": 10250
}
},
"images": [
{
"names": [
"quay.io/calico/node@sha256:7758c25549fcfe677699bbcd3c279b3a174e7cbbbf9d16f3d71713d68f695dfb",
"quay.io/calico/node:v2.6.7"
],
"sizeBytes": 281572001
},
{
"names": [
"protokube:1.10.0-beta.1"
],
"sizeBytes": 277860377
},
{
"names": [
"postgres@sha256:5284ba74a1065e34cf1bfccd64caf8c497c8dc623d6207b060b5ebd369427d34",
"postgres:9.6.2"
],
"sizeBytes": 266926469
},
{
"names": [
"k8s.gcr.io/kube-proxy@sha256:b81228e8ad694f05a5a6e035167ad705600aead5cfd63628a38984fc60f0b989",
"k8s.gcr.io/kube-proxy:v1.10.5"
],
"sizeBytes": 97861102
},
{
"names": [
"quay.io/calico/cni@sha256:3a23e093b1e98cf232a226fedff591d33919f5297f016a41d8012efc83b23a84",
"quay.io/calico/cni:v1.11.2"
],
"sizeBytes": 70781000
},
{
"names": [
"gcr.io/kubernetes-helm/tiller@sha256:2a3dd484ecfcf9343994e0f6c2af0a6faf1af7f7e499905793643f91e90edcb3",
"gcr.io/kubernetes-helm/tiller:v2.10.0"
],
"sizeBytes": 68943449
},
{
"names": [
"k8s.gcr.io/coredns@sha256:db2bf53126ed1c761d5a41f24a1b82a461c85f736ff6e90542e9522be4757848",
"k8s.gcr.io/coredns:1.1.3"
],
"sizeBytes": 45564078
},
{
"names": [
"jtblin/kube2iam@sha256:3fcf3bcd2a400f9d7300a2943bdeb2e130adf693963ecd8dd82de664a9f2967b",
"jtblin/kube2iam:0.10.0"
],
"sizeBytes": 34002320
},
{
"names": [
"k8s.gcr.io/pause-amd64@sha256:163ac025575b775d1c0f9bf0bdd0f086883171eb475b5068e7defa4ca9e76516",
"k8s.gcr.io/pause-amd64:3.0"
],
"sizeBytes": 746888
}
],
"nodeInfo": {
"architecture": "amd64",
"bootID": "cd594eb1-8f8a-4f0d-b24e-c78f22f53430",
"containerRuntimeVersion": "docker://17.3.2",
"kernelVersion": "4.9.0-6-amd64",
"kubeProxyVersion": "v1.10.5",
"kubeletVersion": "v1.10.5",
"machineID": "2c0d85036fb74aaf9c7c29db98b305f8",
"operatingSystem": "linux",
"osImage": "Debian GNU/Linux 9 (stretch)",
"systemUUID": "EC20B3BA-82DC-AC57-288C-7AF592EC538E"
},
"volumesAttached": [
{
"devicePath": "/dev/xvdcg",
"name": "kubernetes.io/aws-ebs/aws://us-east-1a/vol-0a8e02b95e715378e"
}
],
"volumesInUse": [
"kubernetes.io/aws-ebs/aws://us-east-1a/vol-0a8e02b95e715378e"
]
}
},
{
"metadata": {
"annotations": {
"node.alpha.kubernetes.io/ttl": "0",
"volumes.kubernetes.io/controller-managed-attach-detach": "true"
},
"creationTimestamp": "2018-09-03T13:48:30Z",
"labels": {
"beta.kubernetes.io/arch": "amd64",
"beta.kubernetes.io/instance-type": "t2.medium",
"beta.kubernetes.io/os": "linux",
"failure-domain.beta.kubernetes.io/region": "us-east-1",
"failure-domain.beta.kubernetes.io/zone": "us-east-1a",
"kops.k8s.io/instancegroup": "nodes",
"kubernetes.io/hostname": "ip-172-20-48-77.ec2.internal",
"kubernetes.io/role": "node",
"node-role.kubernetes.io/node": ""
},
"name": "ip-172-20-48-77.ec2.internal",
"resourceVersion": "25828",
"selfLink": "/api/v1/nodes/ip-172-20-48-77.ec2.internal",
"uid": "0ad24a1a-af80-11e8-9164-12e606cb7b90"
},
"spec": {
"externalID": "i-066cbfe289cabb845",
"podCIDR": "100.96.6.0/24",
"providerID": "aws:///us-east-1a/i-066cbfe289cabb845"
},
"status": {
"addresses": [
{
"address": "172.20.48.77",
"type": "InternalIP"
},
{
"address": "ip-172-20-48-77.ec2.internal",
"type": "InternalDNS"
},
{
"address": "ip-172-20-48-77.ec2.internal",
"type": "Hostname"
}
],
"allocatable": {
"cpu": "2",
"ephemeral-storage": "115834311284",
"hugepages-2Mi": "0",
"memory": "3946708Ki",
"pods": "110"
},
"capacity": {
"cpu": "2",
"ephemeral-storage": "125688272Ki",
"hugepages-2Mi": "0",
"memory": "4049108Ki",
"pods": "110"
},
"conditions": [
{
"lastHeartbeatTime": "2018-09-03T14:12:55Z",
"lastTransitionTime": "2018-09-03T13:57:44Z",
"message": "kubelet has sufficient disk space available",
"reason": "KubeletHasSufficientDisk",
"status": "False",
"type": "OutOfDisk"
},
{
"lastHeartbeatTime": "2018-09-03T14:12:55Z",
"lastTransitionTime": "2018-09-03T13:57:44Z",
"message": "kubelet has sufficient memory available",
"reason": "KubeletHasSufficientMemory",
"status": "False",
"type": "MemoryPressure"
},
{
"lastHeartbeatTime": "2018-09-03T14:12:55Z",
"lastTransitionTime": "2018-09-03T13:57:44Z",
"message": "kubelet has no disk pressure",
"reason": "KubeletHasNoDiskPressure",
"status": "False",
"type": "DiskPressure"
},
{
"lastHeartbeatTime": "2018-09-03T14:12:55Z",
"lastTransitionTime": "2018-09-03T13:48:30Z",
"message": "kubelet has sufficient PID available",
"reason": "KubeletHasSufficientPID",
"status": "False",
"type": "PIDPressure"
},
{
"lastHeartbeatTime": "2018-09-03T14:12:55Z",
"lastTransitionTime": "2018-09-03T13:57:44Z",
"message": "kubelet is posting ready status",
"reason": "KubeletReady",
"status": "True",
"type": "Ready"
}
],
"daemonEndpoints": {
"kubeletEndpoint": {
"Port": 10250
}
},
"images": [
{
"names": [
"quay.io/calico/node@sha256:7758c25549fcfe677699bbcd3c279b3a174e7cbbbf9d16f3d71713d68f695dfb",
"quay.io/calico/node:v2.6.7"
],
"sizeBytes": 281572001
},
{
"names": [
"protokube:1.10.0-beta.1"
],
"sizeBytes": 277860377
},
{
"names": [
"k8s.gcr.io/kube-proxy@sha256:b81228e8ad694f05a5a6e035167ad705600aead5cfd63628a38984fc60f0b989",
"k8s.gcr.io/kube-proxy:v1.10.5"
],
"sizeBytes": 97861102
},
{
"names": [
"quay.io/calico/cni@sha256:3a23e093b1e98cf232a226fedff591d33919f5297f016a41d8012efc83b23a84",
"quay.io/calico/cni:v1.11.2"
],
"sizeBytes": 70781000
},
{
"names": [
"quay.io/calico/kube-controllers@sha256:e60bdbab2f57540e68f9e8b1a585913e14f0e8ff0628f1e42ad69e4bfb706840",
"quay.io/calico/kube-controllers:v1.0.3"
],
"sizeBytes": 52247098
},
{
"names": [
"jtblin/kube2iam@sha256:3fcf3bcd2a400f9d7300a2943bdeb2e130adf693963ecd8dd82de664a9f2967b",
"jtblin/kube2iam:0.10.0"
],
"sizeBytes": 34002320
},
{
"names": [
"alpine@sha256:56e2f91ef15847a2b02a5a03cbfa483949d67a242c37e33ea178e3e7e01e0dfd",
"alpine:3.7"
],
"sizeBytes": 4197490
},
{
"names": [
"k8s.gcr.io/pause-amd64@sha256:163ac025575b775d1c0f9bf0bdd0f086883171eb475b5068e7defa4ca9e76516",
"k8s.gcr.io/pause-amd64:3.0"
],
"sizeBytes": 746888
}
],
"nodeInfo": {
"architecture": "amd64",
"bootID": "e3282b8b-b8f8-43c5-a05e-d338bb769036",
"containerRuntimeVersion": "docker://17.3.2",
"kernelVersion": "4.9.0-6-amd64",
"kubeProxyVersion": "v1.10.5",
"kubeletVersion": "v1.10.5",
"machineID": "560276c5868740d7acd8f0b09a493741",
"operatingSystem": "linux",
"osImage": "Debian GNU/Linux 9 (stretch)",
"systemUUID": "EC29EE95-6541-C2ED-56B9-DC63778A26FF"
},
"volumesAttached": [
{
"devicePath": "/dev/xvdbr",
"name": "kubernetes.io/aws-ebs/aws://us-east-1a/vol-03e5cb9933e99472a"
},
{
"devicePath": "/dev/xvdcy",
"name": "kubernetes.io/aws-ebs/aws://us-east-1a/vol-0037b06f2bdbb6ce5"
}
],
"volumesInUse": [
"kubernetes.io/aws-ebs/aws://us-east-1a/vol-0037b06f2bdbb6ce5",
"kubernetes.io/aws-ebs/aws://us-east-1a/vol-03e5cb9933e99472a"
]
}
}
],
"kind": "NodeList",
"metadata": {
"resourceVersion": "25830",
"selfLink": "/api/v1/nodes"
}
}
I0903 15:12:56.974690 15573 s3fs.go:256] Listing objects in S3 bucket "example-org-cluster-config" with prefix "shine.dev.example.org/instancegroup/"
I0903 15:12:57.083560 15573 s3fs.go:284] Listed files in s3://example-org-cluster-config/shine.dev.example.org/instancegroup: [s3://example-org-cluster-config/shine.dev.example.org/instancegroup/master-us-east-1a s3://example-org-cluster-config/shine.dev.example.org/instancegroup/nodes]
I0903 15:12:57.083592 15573 s3fs.go:219] Reading file "s3://example-org-cluster-config/shine.dev.example.org/instancegroup/master-us-east-1a"
I0903 15:12:57.190497 15573 s3fs.go:219] Reading file "s3://example-org-cluster-config/shine.dev.example.org/instancegroup/nodes"
I0903 15:12:57.310167 15573 aws_cloud.go:984] Querying EC2 for all valid zones in region "us-east-1"
I0903 15:12:57.310279 15573 request_logger.go:45] AWS request: ec2/DescribeAvailabilityZones
I0903 15:12:58.031110 15573 aws_cloud.go:424] Listing all Autoscaling groups matching cluster tags
I0903 15:12:58.031236 15573 request_logger.go:45] AWS request: autoscaling/DescribeTags
I0903 15:12:58.552427 15573 request_logger.go:45] AWS request: autoscaling/DescribeAutoScalingGroups
NAME STATUS NEEDUPDATE READY MIN MAX NODES
master-us-east-1a NeedsUpdate 1 0 1 1 1
nodes NeedsUpdate 2 0 2 2 2
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment