Hi, just picking up were they left off.
We've made the changes. But issue still persists. Here is details.
1. Router deployed using terraform-helm
ctrl.endpoint=var.ctrlEndpoint=ztnctrl.domain.co:443
resource "helm_release" "router" {
name = var.routerName
repository = "https://openziti.github.io/helm-charts/"
chart = "ziti-router"
version = "1.0.4"
namespace = "ziti"
create_namespace = true
set {
name = "enrollmentJwt"
value = var.routerjwt
}
set {
name = "advertisedHost"
value = format("%s.ziti.svc.cluster.local", var.routerName)
}
set {
name = "linkListeners.transport.service.enabled"
value = "false"
}
set {
name = "ctrl.endpoint"
value = var.ctrlEndpoint
}
set {
name = "tunnel.mode"
value = "host"
}
depends_on = [ kubernetes_secret.cloudflare_api_token_secret ]
}
$ kubectl -n ziti get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
demo-router-edge ClusterIP 10.3.129.152 <none> 443/TCP 7m12s
$ kubectl -n ziti get po
NAME READY STATUS RESTARTS AGE
demo-router-6bfd565dcb-mjrhs 1/1 Running 0 7m28s
$ kubectl -n ziti logs demo-router-6bfd565dcb-mjrhs
{"file":"github.com/openziti/ziti/router/enroll/enroll.go:206","func":"github.com/openziti/ziti/router/enroll.(*RestEnroller).Enroll","level":"info","msg":"registration complete","time":"2024-07-18T02:57:31.684Z"}
{"arch":"amd64","build-date":"2024-05-30T16:36:13Z","configFile":"/etc/ziti/config/ziti-router.yaml","file":"github.com/openziti/ziti/ziti/router/run.go:71","func":"github.com/openziti/ziti/ziti/router.run","go-version":"go1.22.3","level":"info","msg":"starting ziti router","os":"linux","revision":"82c4a7125227","routerId":"1ojTGASXwL","time":"2024-07-18T02:57:31.724Z","version":"v1.1.3"}
{"file":"github.com/openziti/ziti/common/metrics/pool_metrics.go:50","func":"github.com/openziti/ziti/common/metrics.ConfigureGoroutinesPoolMetrics.GoroutinesPoolMetricsConfigF.func1.1","idleTime":30000000000,"level":"info","maxQueueSize":1000,"maxWorkers":32,"minWorkers":0,"msg":"starting goroutine pool","poolType":"pool.link.dialer","time":"2024-07-18T02:57:31.725Z"}
{"file":"github.com/openziti/ziti/common/metrics/pool_metrics.go:50","func":"github.com/openziti/ziti/common/metrics.ConfigureGoroutinesPoolMetrics.GoroutinesPoolMetricsConfigF.func1.1","idleTime":30000000000,"level":"info","maxQueueSize":1000,"maxWorkers":128,"minWorkers":0,"msg":"starting goroutine pool","poolType":"pool.route.handler","time":"2024-07-18T02:57:31.725Z"}
{"file":"github.com/openziti/ziti/common/metrics/pool_metrics.go:50","func":"github.com/openziti/ziti/common/metrics.ConfigureGoroutinesPoolMetrics.GoroutinesPoolMetricsConfigF.func1.1","idleTime":30000000000,"level":"info","maxQueueSize":1,"maxWorkers":50,"minWorkers":0,"msg":"starting goroutine pool","poolType":"pool.terminator_validation","time":"2024-07-18T02:57:31.725Z"}
{"file":"github.com/openziti/ziti/router/internal/edgerouter/config.go:154","func":"github.com/openziti/ziti/router/internal/edgerouter.(*Config).LoadConfigFromMap","level":"info","msg":"cached data model file set to: /etc/ziti/config/ziti-router.yaml.json.gzip","time":"2024-07-18T02:57:31.725Z"}
{"file":"github.com/openziti/ziti/router/internal/edgerouter/config.go:171","func":"github.com/openziti/ziti/router/internal/edgerouter.(*Config).LoadConfigFromMap","level":"warning","msg":"Invalid heartbeat interval [0] (min: 60, max: 10), setting to default [60]","time":"2024-07-18T02:57:31.725Z"}
{"file":"github.com/openziti/ziti/router/internal/edgerouter/config.go:345","func":"github.com/openziti/ziti/router/internal/edgerouter.parseEdgeListenerOptions","level":"info","msg":"advertised port [0] in [listeners[443].options.advertise] does not match the listening port [0] in [listeners[3022].address].","time":"2024-07-18T02:57:31.725Z"}
{"error":"open /etc/ziti/config/ziti-router.yaml.json.gzip: no such file or directory","file":"github.com/openziti/ziti/router/state/manager.go:213","func":"github.com/openziti/ziti/router/state.(*ManagerImpl).LoadRouterModel","level":"error","msg":"could not load router model from file [/etc/ziti/config/ziti-router.yaml.json.gzip]","time":"2024-07-18T02:57:31.725Z"}
{"file":"github.com/openziti/ziti/router/forwarder/faulter.go:78","func":"github.com/openziti/ziti/router/forwarder.(*Faulter).run","level":"info","msg":"started","time":"2024-07-18T02:57:31.726Z"}
{"file":"github.com/openziti/ziti/router/forwarder/scanner.go:52","func":"github.com/openziti/ziti/router/forwarder.(*Scanner).run","level":"info","msg":"started","time":"2024-07-18T02:57:31.726Z"}
{"file":"github.com/openziti/ziti/router/router.go:346","func":"github.com/openziti/ziti/router.(*Router).showOptions","level":"info","msg":"ctrl = {\"OutQueueSize\":4,\"MaxQueuedConnects\":1,\"MaxOutstandingConnects\":16,\"ConnectTimeout\":5000000000,\"DelayRxStart\":false,\"WriteTimeout\":0}","time":"2024-07-18T02:57:31.726Z"}
{"file":"github.com/openziti/ziti/router/router.go:352","func":"github.com/openziti/ziti/router.(*Router).showOptions","level":"info","msg":"metrics = {\"ReportInterval\":60000000000,\"IntervalAgeThreshold\":0,\"MessageQueueSize\":10}","time":"2024-07-18T02:57:31.727Z"}
{"file":"github.com/openziti/ziti/common/metrics/pool_metrics.go:50","func":"github.com/openziti/ziti/common/metrics.ConfigureGoroutinesPoolMetrics.GoroutinesPoolMetricsConfigF.func1.1","idleTime":30000000000,"level":"info","maxQueueSize":5000,"maxWorkers":64,"minWorkers":0,"msg":"starting goroutine pool","poolType":"pool.rate_limiter","time":"2024-07-18T02:57:31.727Z"}
{"file":"github.com/openziti/ziti/router/router.go:660","func":"github.com/openziti/ziti/router.(*Router).initializeHealthChecks","level":"info","msg":"starting health check with ctrl ping initially after 15s, then every 30s, timing out after 15s","time":"2024-07-18T02:57:31.727Z"}
{"file":"github.com/openziti/ziti/router/router.go:481","func":"github.com/openziti/ziti/router.(*Router).startXlinkDialers","level":"info","msg":"started Xlink dialer with binding [transport]","time":"2024-07-18T02:57:31.727Z"}
{"address":{},"file":"github.com/openziti/ziti/router/xgress_edge/listener.go:87","func":"github.com/openziti/ziti/router/xgress_edge.(*listener).Listen","level":"info","msg":"starting channel listener","time":"2024-07-18T02:57:31.727Z"}
{"file":"github.com/openziti/ziti/common/metrics/pool_metrics.go:50","func":"github.com/openziti/ziti/router/xgress_edge.(*listener).Listen.GoroutinesPoolMetricsConfigF.func1.1","idleTime":10000000000,"level":"info","maxQueueSize":1,"maxWorkers":16,"minWorkers":1,"msg":"starting goroutine pool","poolType":"pool.listener.xgress_edge","time":"2024-07-18T02:57:31.727Z"}
{"file":"github.com/openziti/ziti/router/router.go:544","func":"github.com/openziti/ziti/router.(*Router).startXgressListeners","level":"info","msg":"created xgress listener [edge] at [tls:0.0.0.0:3022]","time":"2024-07-18T02:57:31.727Z"}
{"file":"github.com/openziti/ziti/router/router.go:544","func":"github.com/openziti/ziti/router.(*Router).startXgressListeners","level":"info","msg":"created xgress listener [tunnel] at []","time":"2024-07-18T02:57:31.727Z"}
{"file":"github.com/openziti/ziti/router/router.go:722","func":"github.com/openziti/ziti/router.(*Router).getInitialCtrlEndpoints","level":"info","msg":"controller endpoints file [/etc/ziti/config/endpoints] doesn't exist. Using initial endpoints from config","time":"2024-07-18T02:57:31.727Z"}
{"file":"github.com/openziti/ziti/router/router.go:555","func":"github.com/openziti/ziti/router.(*Router).startControlPlane","level":"info","msg":"router configured with 1 controller endpoints","time":"2024-07-18T02:57:31.727Z"}
{"endpoint":{"tls:ztnctrl.domain.co:443":{}},"file":"github.com/openziti/ziti/router/env/ctrls.go:93","func":"github.com/openziti/ziti/router/env.(*networkControllers).UpdateControllerEndpoints","level":"info","msg":"adding new ctrl endpoint","time":"2024-07-18T02:57:31.727Z"}
{"endpoint":"tls:ztnctrl.domain.co:443","file":"github.com/openziti/ziti/router/env/ctrls.go:132","func":"github.com/openziti/ziti/router/env.(*networkControllers).connectToControllerWithBackoff","level":"info","msg":"starting connection attempts","time":"2024-07-18T02:57:31.727Z"}
{"file":"github.com/openziti/ziti/router/xgress_edge/accept.go:126","func":"github.com/openziti/ziti/router/xgress_edge.(*Acceptor).Run","level":"info","msg":"starting","time":"2024-07-18T02:57:31.727Z"}
{"endpoint":"tls:ztnctrl.domain.co:443","file":"github.com/openziti/ziti/router/env/ctrls.go:138","func":"github.com/openziti/ziti/router/env.(*networkControllers).connectToControllerWithBackoff.func3","level":"info","msg":"successfully connected to controller","time":"2024-07-18T02:57:32.212Z"}
{"ctrlId":"ziti-controller-ctrl-plane-identity","file":"github.com/openziti/ziti/router/link/link_registry.go:306","func":"github.com/openziti/ziti/router/link.(*linkRegistryImpl).NotifyOfReconnect","level":"info","msg":"resending link states after reconnect","time":"2024-07-18T02:57:32.212Z"}
{"file":"github.com/openziti/ziti/router/xgress_edge/factory.go:76","func":"github.com/openziti/ziti/router/xgress_edge.(*Factory).NotifyOfReconnect","level":"info","msg":"control channel reconnected, re-establishing hosted services","time":"2024-07-18T02:57:32.212Z"}
{"file":"github.com/openziti/ziti/router/xgress_edge_tunnel/factory.go:56","func":"github.com/openziti/ziti/router/xgress_edge_tunnel.(*Factory).NotifyOfReconnect","level":"info","msg":"control channel reconnected, re-establishing hosted services","time":"2024-07-18T02:57:32.212Z"}
{"file":"github.com/openziti/ziti/router/handler_edge_ctrl/hello.go:82","func":"github.com/openziti/ziti/router/handler_edge_ctrl.(*helloHandler).HandleReceive.func1","level":"info","msg":"received server hello, replying","time":"2024-07-18T02:57:32.213Z"}
{"file":"github.com/openziti/ziti/router/state/apiSessionAdded.go:203","func":"github.com/openziti/ziti/router/state.(*apiSessionAddedHandler).instantSync","level":"info","msg":"first api session syncId [clyqokwochgxz0d9t7xwk0bc0], starting","strategy":"instant","time":"2024-07-18T02:57:32.338Z"}
{"file":"github.com/openziti/ziti/router/state/apiSessionAdded.go:268","func":"github.com/openziti/ziti/router/state.(*apiSessionSyncTracker).Add","level":"info","msg":"received api session sync chunk 0, isLast=true","time":"2024-07-18T02:57:32.339Z"}
{"file":"github.com/openziti/ziti/router/state/manager.go:604","func":"github.com/openziti/ziti/router/state.(*ManagerImpl).StartHeartbeat","level":"info","msg":"heartbeat starting","time":"2024-07-18T02:57:32.362Z"}
{"file":"github.com/openziti/ziti/router/xgress_edge_tunnel/tunneler.go:71","func":"github.com/openziti/ziti/router/xgress_edge_tunnel.(*tunneler).Start","level":"info","mode":"host","msg":"creating interceptor","time":"2024-07-18T02:57:32.362Z"}
{"error":"exec: \"resolvectl\": executable file not found in $PATH","file":"github.com/openziti/ziti/tunnel/dns/server.go:49","func":"github.com/openziti/ziti/tunnel/dns.flushDnsCaches","level":"warning","msg":"unable to find systemd-resolve or resolvectl in path, consider adding a dns flush to your restart process","time":"2024-07-18T02:57:32.362Z"}
{"file":"github.com/openziti/ziti/tunnel/dns/server.go:89","func":"github.com/openziti/ziti/tunnel/dns.NewDnsServer","level":"info","msg":"starting dns server...","time":"2024-07-18T02:57:32.362Z"}
{"file":"github.com/openziti/ziti/router/xgress_edge/certchecker.go:124","func":"github.com/openziti/ziti/router/xgress_edge.(*CertExpirationChecker).Run","level":"info","msg":"waiting 0s to renew certificates","time":"2024-07-18T02:57:32.362Z"}
{"error":"dns server failed to start: listen udp 127.0.0.1:53: bind: permission denied","file":"github.com/openziti/ziti/router/xgress_edge_tunnel/tunneler.go:75","func":"github.com/openziti/ziti/router/xgress_edge_tunnel.(*tunneler).Start","level":"error","msg":"failed to start DNS resolver. using dummy resolver","time":"2024-07-18T02:57:32.362Z"}
{"file":"github.com/openziti/ziti/tunnel/dns/dummy.go:37","func":"github.com/openziti/ziti/tunnel/dns.NewDummyResolver","level":"warning","msg":"dummy resolver does not store hostname/ip mappings","time":"2024-07-18T02:57:32.362Z"}
{"file":"github.com/openziti/ziti/tunnel/intercept/iputils.go:51","func":"github.com/openziti/ziti/tunnel/intercept.SetDnsInterceptIpRange","level":"info","msg":"dns intercept IP range: 100.64.0.1 - 100.127.255.255","time":"2024-07-18T02:57:32.362Z"}
{"file":"github.com/openziti/ziti/tunnel/intercept/svcpoll.go:155","func":"github.com/openziti/ziti/tunnel/intercept.(*ServiceListener).HandleServicesChange","level":"info","msg":"adding service","service":"hello-service","time":"2024-07-18T02:57:32.629Z"}
{"file":"github.com/openziti/ziti/tunnel/intercept/svcpoll.go:248","func":"github.com/openziti/ziti/tunnel/intercept.(*ServiceListener).addService","level":"info","msg":"Hosting newly available service","serviceId":"EBDmwwy8tvOBF3iO8BQ09","serviceName":"hello-service","time":"2024-07-18T02:57:32.629Z"}
{"file":"github.com/openziti/ziti/router/xgress_edge_tunnel/fabric.go:396","func":"github.com/openziti/ziti/router/xgress_edge_tunnel.(*fabricProvider).establishTerminatorWithRetry.func1","level":"info","msg":"attempting to establish terminator","service":"hello-service","time":"2024-07-18T02:57:32.629Z"}
{"channel":"ctrl","file":"github.com/openziti/ziti/router/handler_edge_ctrl/extendEnrollmentCerts.go:126","fingerprint":"90b2c026bc11173c073ac5755be37c0a0f1df3f2","func":"github.com/openziti/ziti/router/handler_edge_ctrl.(*extendEnrollmentCertsHandler).HandleReceive.func1","level":"info","msg":"enrollment extension done","newFingerprint":"76a5874a5d4a1972b7d324bd949ac5efb7bef468","time":"2024-07-18T02:57:32.634Z"}
{"file":"github.com/openziti/ziti/router/xgress_edge/certchecker.go:124","func":"github.com/openziti/ziti/router/xgress_edge.(*CertExpirationChecker).Run","level":"info","msg":"waiting 8591h59m59.3655268s to renew certificates","time":"2024-07-18T02:57:32.634Z"}
{"file":"github.com/openziti/ziti/router/xgress_edge_tunnel/fabric.go:487","func":"github.com/openziti/ziti/router/xgress_edge_tunnel.(*fabricProvider).HandleTunnelResponse","level":"info","msg":"received new session","routerId":"1ojTGASXwL","sessionId":"clyqokwzwhgy20d9t41lqzb9j","terminatorId":"4D425VpefAyJaVNK48LzkJ","time":"2024-07-18T02:57:32.759Z"}
{"createDuration":130242501,"file":"github.com/openziti/ziti/router/xgress_edge_tunnel/fabric.go:509","func":"github.com/openziti/ziti/router/xgress_edge_tunnel.(*fabricProvider).HandleTunnelResponse","level":"info","msg":"received terminator created notification","routerId":"1ojTGASXwL","terminatorId":"4D425VpefAyJaVNK48LzkJ","time":"2024-07-18T02:57:32.759Z"}
{"file":"github.com/openziti/ziti/router/state/apiSessionAdded.go:124","func":"github.com/openziti/ziti/router/state.(*apiSessionAddedHandler).applySync","level":"info","msg":"finished synchronizing api sessions [count: 6, syncId: clyqokwochgxz0d9t7xwk0bc0, duration: 24.829µs]","time":"2024-07-18T02:57:33.339Z"}
{"_context":"ch{ctrl}-\u003eu{reconnecting}-\u003ei{8Je96}","file":"github.com/openziti/ziti/router/handler_ctrl/validate_terminators_v2.go:94","func":"github.com/openziti/ziti/router/handler_ctrl.(*validateTerminatorsV2Handler).validateTerminators.func1","level":"info","msg":"validating terminator","terminatorId":"4D425VpefAyJaVNK48LzkJ","time":"2024-07-18T02:58:02.335Z"}
2. ziti resources
$ ziti edge list edge-routers
╭────────────┬─────────────┬────────┬───────────────┬──────┬────────────╮
│ ID │ NAME │ ONLINE │ ALLOW TRANSIT │ COST │ ATTRIBUTES │
├────────────┼─────────────┼────────┼───────────────┼──────┼────────────┤
│ 1ojTGASXwL │ demo-router │ true │ true │ 0 │ │
╰────────────┴─────────────┴────────┴───────────────┴──────┴────────────╯
$ ziti edge list identities
╭────────────┬───────────────┬─────────┬───────────────┬─────────────╮
│ ID │ NAME │ TYPE │ ATTRIBUTES │ AUTH-POLICY │
├────────────┼───────────────┼─────────┼───────────────┼─────────────┤
│ 1ojTGASXwL │ demo-router │ Router │ hello-hosts │ Default │
│ 8jJaPAEXwL │ hello-client │ Default │ hello-clients │ Default │
│ gDiEQCdq6 │ Default Admin │ Default │ │ Default │
╰────────────┴───────────────┴─────────┴───────────────┴─────────────╯
$ ziti edge list configs
╭────────────────────────┬────────────────────────┬──────────────╮
│ ID │ NAME │ CONFIG TYPE │
├────────────────────────┼────────────────────────┼──────────────┤
│ 1LD8tmnfFDDWVFAqiK8Pbl │ hello-host-config │ host.v1 │
│ 2K00XEZssvHN9ZVjACSMZM │ hello-intercept-config │ intercept.v1 │
╰────────────────────────┴────────────────────────┴──────────────╯
$ ziti edge list services
╭───────────────────────┬───────────────┬────────────┬─────────────────────┬────────────╮
│ ID │ NAME │ ENCRYPTION │ TERMINATOR STRATEGY │ ATTRIBUTES │
│ │ │ REQUIRED │ │ │
├───────────────────────┼───────────────┼────────────┼─────────────────────┼────────────┤
│ EBDmwwy8tvOBF3iO8BQ09 │ hello-service │ true │ smartrouting │ │
╰───────────────────────┴───────────────┴────────────┴─────────────────────┴────────────╯
$ ziti edge list edge-router-policies
╭───────────────────────┬───────────────────────────────┬───────────────────┬────────────────╮
│ ID │ NAME │ EDGE ROUTER ROLES │ IDENTITY ROLES │
├───────────────────────┼───────────────────────────────┼───────────────────┼────────────────┤
│ 1ojTGASXwL │ edge-router-1ojTGASXwL-system │ @demo-router │ @demo-router │
│ jzy9aszZwRegEJJyhfGgx │ default │ #all │ #all │
╰───────────────────────┴───────────────────────────────┴───────────────────┴────────────────╯
# ziti edge list service-edge-router-policies
╭────────────────────────┬─────────┬───────────────┬───────────────────╮
│ ID │ NAME │ SERVICE ROLES │ EDGE ROUTER ROLES │
├────────────────────────┼─────────┼───────────────┼───────────────────┤
│ 4bNtqqgBodS3Ybz8fGgn2b │ default │ #all │ #all │
╰────────────────────────┴─────────┴───────────────┴───────────────────╯
$ ziti edge list service-policies
╭────────────────────────┬───────────────────┬──────────┬────────────────┬────────────────┬─────────────────────╮
│ ID │ NAME │ SEMANTIC │ SERVICE ROLES │ IDENTITY ROLES │ POSTURE CHECK ROLES │
├────────────────────────┼───────────────────┼──────────┼────────────────┼────────────────┼─────────────────────┤
│ 1VVFRSdie6Ox7CZRJ0CavZ │ hello-dial-policy │ AllOf │ @hello-service │ #hello-clients │ │
│ S4MsBqv8QHLL2BwzXUiD1 │ hello-bind-policy │ AllOf │ @hello-service │ #hello-hosts │ │
╰────────────────────────┴───────────────────┴──────────┴────────────────┴────────────────┴─────────────────────╯
3. Testing Connection
$ ziti edge policy-advisor services hello-service -q
OKAY : demo-router (1) -> hello-service (1) Common Routers: (1/1) Dial: N Bind: Y
OKAY : hello-client (1) -> hello-service (1) Common Routers: (1/1) Dial: Y Bind: N
$ ls /opt/openziti/etc/identities/
hello-client.json
$ systemctl status ziti-edge-tunnel.service
● ziti-edge-tunnel.service - Ziti Edge Tunnel
Loaded: loaded (/usr/lib/systemd/system/ziti-edge-tunnel.service; enabled; preset: enabled)
Active: active (running) since Thu 2024-07-18 02:39:12 UTC; 36min ago
Process: 526778 ExecStartPre=/opt/openziti/bin/ziti-edge-tunnel.sh (code=exited, status=0/SUCCESS)
Main PID: 526784 (ziti-edge-tunne)
Tasks: 6 (limit: 4586)
Memory: 5.5M (peak: 6.7M)
CPU: 11.734s
CGroup: /system.slice/ziti-edge-tunnel.service
└─526784 /opt/openziti/bin/ziti-edge-tunnel run --verbose=2 --dns-ip-range=100.64.0.1/10 --identity-dir=/opt/openziti/etc/identities
Jul 18 03:04:51 ip-172-31-38-32 ziti-edge-tunnel[526784]: (526784)[ 1538.985] ERROR ziti-sdk:channel.c:903 on_channel_connect_internal() ch[1] failed to connect to ER[demo-router] [-3001/temporary failu>
Jul 18 03:04:57 ip-172-31-38-32 ziti-edge-tunnel[526784]: (526784)[ 1544.792] ERROR ziti-sdk:channel.c:903 on_channel_connect_internal() ch[1] failed to connect to ER[demo-router] [-3001/temporary failu>
Jul 18 03:07:11 ip-172-31-38-32 ziti-edge-tunnel[526784]: (526784)[ 1678.786] ERROR ziti-sdk:channel.c:903 on_channel_connect_internal() ch[1] failed to connect to ER[demo-router] [-3001/temporary failu>
Jul 18 03:08:00 ip-172-31-38-32 ziti-edge-tunnel[526784]: (526784)[ 1727.559] ERROR ziti-sdk:channel.c:903 on_channel_connect_internal() ch[1] failed to connect to ER[demo-router] [-3001/temporary failu>
Jul 18 03:10:01 ip-172-31-38-32 ziti-edge-tunnel[526784]: (526784)[ 1848.177] ERROR ziti-sdk:channel.c:903 on_channel_connect_internal() ch[1] failed to connect to ER[demo-router] [-3001/temporary failu>
Jul 18 03:11:55 ip-172-31-38-32 ziti-edge-tunnel[526784]: (526784)[ 1962.089] ERROR ziti-sdk:channel.c:903 on_channel_connect_internal() ch[1] failed to connect to ER[demo-router] [-3001/temporary failu>
Jul 18 03:12:18 ip-172-31-38-32 ziti-edge-tunnel[526784]: (526784)[ 1985.287] ERROR ziti-sdk:channel.c:903 on_channel_connect_internal() ch[1] failed to connect to ER[demo-router] [-3001/temporary failu>
Jul 18 03:12:24 ip-172-31-38-32 ziti-edge-tunnel[526784]: (526784)[ 1991.357] ERROR ziti-sdk:channel.c:903 on_channel_connect_internal() ch[1] failed to connect to ER[demo-router] [-3001/temporary failu>
Jul 18 03:14:18 ip-172-31-38-32 ziti-edge-tunnel[526784]: (526784)[ 2105.765] ERROR ziti-sdk:channel.c:903 on_channel_connect_internal() ch[1] failed to connect to ER[demo-router] [-3001/temporary failu>
Jul 18 03:15:29 ip-172-31-38-32 ziti-edge-tunnel[526784]: (526784)[ 2176.490] ERROR ziti-sdk:channel.c:903 on_channel_connect_internal() ch[1] failed to connect to ER[demo-router] [-3001/temporary failu>
$ curl -v hello.ziti.internal
* Host hello.ziti.internal:80 was resolved.
* IPv6: (none)
* IPv4: 100.64.0.3
* Trying 100.64.0.3:80...
* connect to 100.64.0.3 port 80 from 100.64.0.1 port 52150 failed: Connection timed out
* Failed to connect to hello.ziti.internal port 80 after 134826 ms: Couldn't connect to server
* Closing connection
curl: (28) Failed to connect to hello.ziti.internal port 80 after 134826 ms: Couldn't connect to server