HI,
I am deploying SS Cluster using Kubernetes.Unable to spin up Aggregator and Leaf Nodes.
sdb-operator pods are created and working.
Error:Reconciler error : failed to get service endpoint (svc-sdb-cluster-ddl): no ingress endpoint found
Another log:attempting to acquire leader lease default/memsql-operator-lock-sdb-cluster...
Logs:
[root@learning-1 ss_kubernetese]# kubectl logs deployment/sdb-operator
2022/10/13 07:35:19 deleg.go:121 {controller.memsql} reconciliation cause: statefulset namespace: "default" clusterName: "sdb-cluster" serviceName: "svc-sdb-cluster-ddl" namespace: "default"
2022/10/13 07:35:19 deleg.go:121 {controller.memsql} reconciliation cause: statefulset namespace: "default" clusterName: "sdb-cluster" serviceName: "svc-sdb-cluster-ddl" namespace: "default"
2022/10/13 07:35:19 deleg.go:121 {controller.memsql} reconciliation cause: statefulset namespace: "default" clusterName: "sdb-cluster" serviceName: "svc-sdb-cluster" namespace: "default"
2022/10/13 07:35:19 deleg.go:121 {controller.memsql} reconciliation cause: statefulset namespace: "default" clusterName: "sdb-cluster" serviceName: "svc-sdb-cluster" namespace: "default"
2022/10/13 08:52:06 logr.go:249 {controller.memsql} Reconciling MemSQL Cluster. Request.Name: "sdb-cluster" Request.Namespace: "default"
2022/10/13 08:52:06 deleg.go:121 {memsql} can't find operator deployment, trying uncached client key: "default/operator-sdb-cluster"
2022/10/13 08:52:06 deleg.go:135 {memsql} can't find operator deployment, metrics service will not be created error: "deployments.apps "operator-sdb-cluster" not found"
2022/10/13 08:52:06 deleg.go:135 {controller.memsql} Reconciler error, will retry after 10m0s: "error" failed to get service endpoint (svc-sdb-cluster-ddl): no ingress endpoint found
[root@learning-1 ss_kubernetese]# kubectl logs deployment/sdb-operator
Found 2 pods, using pod/sdb-operator-564b9d7d97-l6x22
2022/10/13 09:03:20 deleg.go:121 {cmd} Go Version: go1.18.2
2022/10/13 09:03:20 deleg.go:121 {cmd} Go OS/Arch: linux/amd64
2022/10/13 09:03:20 deleg.go:121 {cmd} Operator Version: 3.0.33
2022/10/13 09:03:20 deleg.go:121 {cmd} Commit Hash: db8f5aff
2022/10/13 09:03:20 deleg.go:121 {cmd} Build Time: 2022-09-08T14:43:05Z
2022/10/13 09:03:20 deleg.go:121 {cmd} Options:
2022/10/13 09:03:20 deleg.go:121 {cmd} --cores-per-unit: 8.000000
2022/10/13 09:03:20 deleg.go:121 {cmd} --memory-per-unit: 32.000000
2022/10/13 09:03:20 deleg.go:121 {cmd} --overpack-factor: 0.000000
2022/10/13 09:03:20 deleg.go:121 {cmd} --extra-cidrs: []
2022/10/13 09:03:20 deleg.go:121 {cmd} --external-dns-domain-name: {false }
2022/10/13 09:03:20 deleg.go:121 {cmd} --external-dns-ttl: {false 0}
2022/10/13 09:03:20 deleg.go:121 {cmd} --ssl-secret-name:
2022/10/13 09:03:20 deleg.go:121 {cmd} --merge-service-annotations: true
2022/10/13 09:03:20 deleg.go:121 {cmd} --backup-default-deadline-seconds: 3600
2022/10/13 09:03:20 deleg.go:121 {cmd} --backup-incremental-default-deadline-seconds: 3600
2022/10/13 09:03:20 deleg.go:121 {cmd} --cluster-id: sdb-cluster
2022/10/13 09:03:20 deleg.go:121 {cmd} --fs-group-id: 5555
2022/10/13 09:03:20 deleg.go:121 {controller-runtime.metrics} Metrics server is starting to listen addr: "0.0.0.0:9090"
2022/10/13 09:03:21 deleg.go:121 {cmd} starting manager
2022/10/13 09:03:21 logr.go:249 Starting server kind: "metrics" addr: "[::]:9090" path: "/metrics"
I1013 09:03:21.196052 1 leaderelection.go:248] attempting to acquire leader lease default/memsql-operator-lock-sdb-cluster...
[root@learning-1 4px]# kubectl get pods
NAME READY STATUS RESTARTS AGE
sdb-operator-9565d987-5rg8l 1/1 Running 0 96m
sdb-operator-9565d987-hjt5j 1/1 Running 0 96m
[root@learning-1 4px]# kubectl get memsqlclusters.memsql.com/sdb-cluster
NAME AGGREGATORS LEAVES REDUNDANCY LEVEL AGE
sdb-cluster 0 0 1 124m
[root@learning-1 4px]# kubectl get services
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 135m
svc-sdb-cluster ClusterIP None <none> 3306/TCP 125m
svc-sdb-cluster-ddl LoadBalancer 10.102.231.87 <pending> 3306:30052/TCP 125m
[root@learning-1 ss_kubernetese]# kubectl describe pod
Name: sdb-operator-564b9d7d97-6xs8d
Namespace: default
Priority: 0
Service Account: sdb-operator
Node: learning-2/10.138.0.3
Start Time: Thu, 13 Oct 2022 09:03:18 +0000
Labels: name=sdb-operator
pod-template-hash=564b9d7d97
Annotations: <none>
Status: Running
IP: 10.244.1.32
IPs:
IP: 10.244.1.32
Controlled By: ReplicaSet/sdb-operator-564b9d7d97
Containers:
sdb-operator:
Container ID: containerd://0586b50eef3d95b561ee335de7678ca2826a3958dda2ba5a45976e510b62744f
Image: singlestore/operator:3.0.32-db8f5aff
Image ID: docker.io/memsql/operator@sha256:cd39e13744e57142eff3fe8e3e55dbb4526778b5331cd0bf4d26c9d2f3526031
Port: <none>
Host Port: <none>
Args:
--merge-service-annotations
--fs-group-id
5555
--cluster-id
sdb-cluster
State: Running
Started: Thu, 13 Oct 2022 09:03:21 +0000
Ready: True
Restart Count: 0
Environment:
WATCH_NAMESPACE: default (v1:metadata.namespace)
POD_NAME: sdb-operator-564b9d7d97-6xs8d (v1:metadata.name)
OPERATOR_NAME: sdb-operator
Mounts:
/var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-4c9pf (ro)
Conditions:
Type Status
Initialized True
Ready True
ContainersReady True
PodScheduled True
Volumes:
kube-api-access-4c9pf:
Type: Projected (a volume that contains injected data from multiple sources)
TokenExpirationSeconds: 3607
ConfigMapName: kube-root-ca.crt
ConfigMapOptional: <nil>
DownwardAPI: true
QoS Class: BestEffort
Node-Selectors: <none>
Tolerations: node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Scheduled 6m15s default-scheduler Successfully assigned default/sdb-operator-564b9d7d97-6xs8d to learning-2
Normal Pulling 6m13s kubelet Pulling image "singlestore/operator:3.0.32-db8f5aff"
Normal Pulled 6m12s kubelet Successfully pulled image "singlestore/operator:3.0.32-db8f5aff" in 775.624181ms
Normal Created 6m12s kubelet Created container sdb-operator
Normal Started 6m12s kubelet Started container sdb-operator
Can Someone pls resolve this issue , I have attached logs.
sdb-operator.yaml
[root@learning-1 ss_kubernetese]# cat sdb-operator.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: sdb-operator
labels:
app.kubernetes.io/component: operator
spec:
replicas: 2
selector:
matchLabels:
name: sdb-operator
template:
metadata:
labels:
name: sdb-operator
spec:
serviceAccountName: sdb-operator
containers:
- name: sdb-operator
image: singlestore/operator:3.0.32-db8f5aff
imagePullPolicy: Always
args: [
# Cause the operator to merge rather than replace annotations on services
"--merge-service-annotations",
# Allow the process inside the container to have read/write access to the `/var/lib/memsql` volume.
"--fs-group-id", "5555",
"--cluster-id", "sdb-cluster"
]
env:
- name: WATCH_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: OPERATOR_NAME
value: "sdb-operator"
sdb-cluster.yaml:
[root@learning-1 ss_kubernetese]# cat sdb-cluster.yaml
apiVersion: memsql.com/v1alpha1
kind: MemsqlCluster
metadata:
name: sdb-cluster
spec:
license: <>
adminHashedPassword: "*9177CC8207174BDBB5ED66B2140C75171283F15D"
nodeImage:
repository: singlestore/node
tag: alma-7.8.17-69cee1f1a3
redundancyLevel: 1
serviceSpec:
objectMetaOverrides:
labels:
custom: label
annotations:
custom: annotations
aggregatorSpec:
count: 1
height: 0.5
storageGB: 512
storageClass: standard
objectMetaOverrides:
annotations:
optional: annotation
labels:
optional: label
leafSpec:
count: 1
height: 0.5
storageGB: 1024
storageClass: standard
objectMetaOverrides:
annotations:
optional: annotation
labels:
optional: label
usersSpec:
rootServiceUser: true
This issue is pending long, I tried finding solution but unable to get it.