master/CiaB: Wait for juju services to have open ports - Timeout when waiting for services
Mac Lin
mkl0301 at gmail.com
Wed Jan 4 12:48:27 UTC 2017
This is the content of juju-bad.log, result of "juju status" on my server.
Or please let me know what command should I gave.
It's difficult for me to try them out in ansible playbook. *sigh*
environment: manual
machines:
"0":
agent-state: started
agent-state-info: (started)
agent-version: 1.25.9
dns-name: juju.cord.lab
instance-id: 'manual:'
series: trusty
hardware: arch=amd64 cpu-cores=8 mem=24112M
state-server-member-status: has-vote
"1":
agent-state: started
agent-state-info: (started)
agent-version: 1.25.9
dns-name: keystone.cord.lab
instance-id: manual:keystone.cord.lab
series: trusty
hardware: arch=amd64 cpu-cores=8 mem=24112M
"2":
agent-state: started
agent-state-info: (started)
agent-version: 1.25.9
dns-name: percona-cluster.cord.lab
instance-id: manual:percona-cluster.cord.lab
series: trusty
hardware: arch=amd64 cpu-cores=8 mem=24112M
"3":
agent-state: started
agent-state-info: (started)
agent-version: 1.25.9
dns-name: nagios.cord.lab
instance-id: manual:nagios.cord.lab
series: trusty
hardware: arch=amd64 cpu-cores=8 mem=24112M
"4":
agent-state: started
agent-state-info: (started)
agent-version: 1.25.9
dns-name: neutron-api.cord.lab
instance-id: manual:neutron-api.cord.lab
series: trusty
hardware: arch=amd64 cpu-cores=8 mem=24112M
"5":
agent-state: started
agent-state-info: (started)
agent-version: 1.25.9
dns-name: nova-cloud-controller.cord.lab
instance-id: manual:nova-cloud-controller.cord.lab
series: trusty
hardware: arch=amd64 cpu-cores=8 mem=24112M
"6":
agent-state: started
agent-state-info: (started)
agent-version: 1.25.9
dns-name: openstack-dashboard.cord.lab
instance-id: manual:openstack-dashboard.cord.lab
series: trusty
hardware: arch=amd64 cpu-cores=8 mem=24112M
"7":
agent-state: started
agent-state-info: (started)
agent-version: 1.25.9
dns-name: rabbitmq-server.cord.lab
instance-id: manual:rabbitmq-server.cord.lab
series: trusty
hardware: arch=amd64 cpu-cores=8 mem=24112M
"8":
agent-state: started
agent-state-info: (started)
agent-version: 1.25.9
dns-name: mongodb.cord.lab
instance-id: manual:mongodb.cord.lab
series: trusty
hardware: arch=amd64 cpu-cores=8 mem=24112M
"9":
agent-state: started
agent-state-info: (started)
agent-version: 1.25.9
dns-name: ceilometer.cord.lab
instance-id: manual:ceilometer.cord.lab
series: trusty
hardware: arch=amd64 cpu-cores=8 mem=24112M
"10":
agent-state: started
agent-state-info: (started)
agent-version: 1.25.9
dns-name: glance.cord.lab
instance-id: manual:glance.cord.lab
series: trusty
hardware: arch=amd64 cpu-cores=8 mem=24112M
services:
ceilometer:
charm: cs:trusty/ceilometer-17
exposed: false
service-status:
current: blocked
message: 'Missing relations: messaging, identity, database'
since: 03 Jan 2017 19:39:55Z
relations:
amqp:
- rabbitmq-server
ceilometer-service:
- ceilometer-agent
cluster:
- ceilometer
identity-service:
- keystone
juju-info:
- nagios
nrpe-external-master:
- nrpe
shared-db:
- mongodb
units:
ceilometer/0:
workload-status:
current: blocked
message: 'Missing relations: messaging, identity, database'
since: 03 Jan 2017 19:39:55Z
agent-status:
current: executing
message: running install hook
since: 03 Jan 2017 14:53:31Z
version: 1.25.9
agent-state: started
agent-version: 1.25.9
machine: "9"
open-ports:
- 8777/tcp
public-address: ceilometer.cord.lab
ceilometer-agent:
charm: cs:trusty/ceilometer-agent-13
exposed: false
service-status: {}
relations:
ceilometer-service:
- ceilometer
glance:
charm: cs:trusty/glance-28
exposed: false
service-status:
current: blocked
message: 'Missing relations: identity, database'
since: 03 Jan 2017 19:39:50Z
relations:
cluster:
- glance
identity-service:
- keystone
image-service:
- nova-cloud-controller
nrpe-external-master:
- nrpe
shared-db:
- percona-cluster
units:
glance/0:
workload-status:
current: blocked
message: 'Missing relations: identity, database'
since: 03 Jan 2017 19:39:50Z
agent-status:
current: executing
message: running install hook
since: 03 Jan 2017 14:54:12Z
version: 1.25.9
agent-state: started
agent-version: 1.25.9
machine: "10"
public-address: glance.cord.lab
keystone:
charm: cs:trusty/keystone-33
exposed: false
service-status:
current: blocked
message: 'Missing relations: database'
since: 03 Jan 2017 19:38:39Z
relations:
cluster:
- keystone
identity-service:
- ceilometer
- glance
- neutron-api
- nova-cloud-controller
- openstack-dashboard
nrpe-external-master:
- nrpe
shared-db:
- percona-cluster
units:
keystone/0:
workload-status:
current: blocked
message: 'Missing relations: database'
since: 03 Jan 2017 19:38:39Z
agent-status:
current: executing
message: running install hook
since: 03 Jan 2017 14:55:09Z
version: 1.25.9
agent-state: started
agent-version: 1.25.9
machine: "1"
public-address: keystone.cord.lab
mongodb:
charm: cs:trusty/mongodb-33
exposed: false
service-status:
current: unknown
message: Waiting for agent initialization to finish
since: 03 Jan 2017 15:13:23Z
relations:
database:
- ceilometer
replica-set:
- mongodb
nagios:
charm: cs:trusty/nagios-10
exposed: false
service-status:
current: maintenance
message: installing charm software
since: 03 Jan 2017 15:00:30Z
relations:
monitors:
- nrpe
nagios:
- ceilometer
units:
nagios/0:
workload-status:
current: maintenance
message: installing charm software
since: 03 Jan 2017 15:00:30Z
agent-status:
current: executing
message: running install hook
since: 03 Jan 2017 15:00:34Z
version: 1.25.9
agent-state: pending
agent-version: 1.25.9
machine: "3"
open-ports:
- 80/tcp
public-address: nagios.cord.lab
neutron-api:
charm: cs:~cordteam/trusty/neutron-api-4
exposed: false
service-status:
current: blocked
message: 'Missing relations: messaging, identity, database'
since: 03 Jan 2017 19:38:45Z
relations:
amqp:
- rabbitmq-server
cluster:
- neutron-api
identity-service:
- keystone
neutron-api:
- nova-cloud-controller
nrpe-external-master:
- nrpe
shared-db:
- percona-cluster
units:
neutron-api/0:
workload-status:
current: blocked
message: 'Missing relations: messaging, identity, database'
since: 03 Jan 2017 19:38:45Z
agent-status:
current: executing
message: running install hook
since: 03 Jan 2017 15:04:12Z
version: 1.25.9
agent-state: started
agent-version: 1.25.9
machine: "4"
open-ports:
- 9696/tcp
public-address: neutron-api.cord.lab
nova-cloud-controller:
charm: cs:trusty/nova-cloud-controller-64
exposed: false
service-status:
current: unknown
message: Waiting for agent initialization to finish
since: 03 Jan 2017 15:03:03Z
relations:
amqp:
- rabbitmq-server
cluster:
- nova-cloud-controller
identity-service:
- keystone
image-service:
- glance
neutron-api:
- neutron-api
nrpe-external-master:
- nrpe
shared-db:
- percona-cluster
units:
nova-cloud-controller/0:
workload-status:
current: unknown
message: Waiting for agent initialization to finish
since: 03 Jan 2017 15:03:03Z
agent-status:
current: allocating
since: 03 Jan 2017 15:03:03Z
agent-state: pending
nrpe:
charm: cs:trusty/nrpe-4
exposed: false
service-status: {}
relations:
general-info:
- percona-cluster
monitors:
- nagios
nrpe-external-master:
- ceilometer
- glance
- keystone
- neutron-api
- nova-cloud-controller
- openstack-dashboard
- rabbitmq-server
subordinate-to:
- ceilometer
- glance
- keystone
- neutron-api
- nova-cloud-controller
- openstack-dashboard
- percona-cluster
- rabbitmq-server
ntp:
charm: cs:trusty/ntp-14
exposed: false
service-status: {}
relations:
ntp-peers:
- ntp
openstack-dashboard:
charm: cs:trusty/openstack-dashboard-19
exposed: false
service-status:
current: unknown
message: Waiting for agent initialization to finish
since: 03 Jan 2017 15:06:02Z
relations:
cluster:
- openstack-dashboard
identity-service:
- keystone
nrpe-external-master:
- nrpe
units:
openstack-dashboard/0:
workload-status:
current: unknown
message: Waiting for agent initialization to finish
since: 03 Jan 2017 15:06:02Z
agent-status:
current: allocating
since: 03 Jan 2017 15:06:02Z
version: 1.25.9
agent-state: pending
agent-version: 1.25.9
machine: "6"
public-address: openstack-dashboard.cord.lab
percona-cluster:
charm: cs:trusty/percona-cluster-31
exposed: false
service-status:
current: active
message: Unit is ready
since: 03 Jan 2017 17:47:40Z
relations:
cluster:
- percona-cluster
juju-info:
- nrpe
shared-db:
- glance
- keystone
- neutron-api
- nova-cloud-controller
units:
percona-cluster/0:
workload-status:
current: active
message: Unit is ready
since: 03 Jan 2017 17:47:40Z
agent-status:
current: executing
message: running install hook
since: 03 Jan 2017 14:56:46Z
version: 1.25.9
agent-state: started
agent-version: 1.25.9
machine: "2"
public-address: percona-cluster.cord.lab
rabbitmq-server:
charm: cs:trusty/rabbitmq-server-42
exposed: false
service-status:
current: unknown
message: Waiting for agent initialization to finish
since: 03 Jan 2017 23:06:51Z
relations:
amqp:
- ceilometer
- neutron-api
- nova-cloud-controller
cluster:
- rabbitmq-server
nrpe-external-master:
- nrpe
units:
rabbitmq-server/0:
workload-status:
current: unknown
message: Waiting for agent initialization to finish
since: 03 Jan 2017 23:06:51Z
agent-status:
current: allocating
since: 03 Jan 2017 23:06:51Z
version: 1.25.9
agent-state: pending
agent-version: 1.25.9
machine: "7"
public-address: rabbitmq-server.cord.lab
On Wed, Jan 4, 2017 at 7:32 PM, Rick Harding <rick.harding at canonical.com>
wrote:
> Can you paste the relations part of the juju status output that shows
> those relations in place? Perhaps there's an issue with the charm logic and
> breaking and re-adding the relations might kick it into gear?
>
>
>
> On Wed, Jan 4, 2017 at 2:48 AM Mac Lin <mkl0301 at gmail.com> wrote:
>
>> Really appreciate for the help. Have been stuck here for weeks.
>>
>> AFAIK, no. I attached two result of "juju status" on CloudLab(good) and
>> my server (bad). It's supposed to work without expose the ports. And I just
>> noticed that on my server, most of the services are in blocked status. But
>> it seems the missing parts are fulfilled. For example, the ceilometer is
>> complaining missing messaging, identity, and database relations, but the
>> relation does exist.
>>
>> Please let me know if any info needed.
>>
>> ceilometer:
>> charm: cs:trusty/ceilometer-17
>> exposed: false
>> service-status:
>> current: blocked
>> message: 'Missing relations: messaging, identity, database'
>> since: 03 Jan 2017 19:39:55Z
>> relations:
>> amqp:
>> - rabbitmq-server
>> ceilometer-service:
>> - ceilometer-agent
>> cluster:
>> - ceilometer
>> identity-service:
>> - keystone
>> juju-info:
>> - nagios
>> nrpe-external-master:
>> - nrpe
>> shared-db:
>> - mongodb
>> units:
>> ceilometer/0:
>> workload-status:
>> current: blocked
>> message: 'Missing relations: messaging, identity, database'
>> since: 03 Jan 2017 19:39:55Z
>> agent-status:
>> current: executing
>> message: running install hook
>> since: 03 Jan 2017 14:53:31Z
>> version: 1.25.9
>> agent-state: started
>> agent-version: 1.25.9
>> machine: "9"
>> open-ports:
>> - 8777/tcp
>> public-address: ceilometer.cord.lab
>>
>>
>>
>> On Tue, Jan 3, 2017 at 8:23 PM, Rick Harding <rick.harding at canonical.com>
>> wrote:
>>
>> Has juju expose been run on the applications? The charm can declare that
>> these ports are the ones that are opened if exposed, it's not actually set
>> until the operator runs the juju expose command on the application.
>>
>> On Sun, Jan 1, 2017 at 2:58 AM Mac Lin <mkl0301 at gmail.com> wrote:
>>
>>
>> Hi,
>>
>> I'm running CORD master/cord-in-a-box.sh on an x86_64 server. The same
>> script on CloudLab has no problem.
>>
>> If I log into each failed service(lxc), I could found the port is not
>> listened because service is not up, which is due to not configured
>> properly. The configuration files are mostly at its initial state.
>>
>> How could I let juju to generate the configuration for the services
>> manually?
>>
>> TASK [juju-finish : Wait for juju services to have open ports]
>> *****************
>> Saturday 31 December 2016 12:04:22 +0000 (0:00:04.738) 0:09:13.981
>> *****
>> failed: [10.100.198.201] (item={u'service': u'ceilometer',
>> u'ipv4_last_octet': 20, u'name': u'ceilometer-1', u'forwarded_ports':
>> [{u'int': 8777, u'ext': 8777}], u'aliase
>> s': [u'ceilometer']}) => {"elapsed": 1800, "failed": true, "item":
>> {"aliases": ["ceilometer"], "forwarded_ports": [{"ext": 8777, "int":
>> 8777}], "ipv4_last_octet": 20, "n
>> ame": "ceilometer-1", "service": "ceilometer"}, "msg": "Timeout when
>> waiting for ceilometer-1:8777"}
>> failed: [10.100.198.201] (item={u'service': u'glance',
>> u'ipv4_last_octet': 30, u'name': u'glance-1', u'forwarded_ports': [{u'int':
>> 9292, u'ext': 9292}], u'aliases': [u'g
>> lance']}) => {"elapsed": 1800, "failed": true, "item": {"aliases":
>> ["glance"], "forwarded_ports": [{"ext": 9292, "int": 9292}],
>> "ipv4_last_octet": 30, "name": "glance-1"
>> , "service": "glance"}, "msg": "Timeout when waiting for glance-1:9292"}
>> failed: [10.100.198.201] (item={u'service': u'keystone',
>> u'ipv4_last_octet': 40, u'name': u'keystone-1', u'forwarded_ports':
>> [{u'int': 35357, u'ext': 35357}, {u'int': 49
>> 90, u'ext': 4990}, {u'int': 5000, u'ext': 5000}], u'aliases':
>> [u'keystone']}) => {"elapsed": 1800, "failed": true, "item": {"aliases":
>> ["keystone"], "forwarded_ports": [
>> {"ext": 35357, "int": 35357}, {"ext": 4990, "int": 4990}, {"ext": 5000,
>> "int": 5000}], "ipv4_last_octet": 40, "name": "keystone-1", "service":
>> "keystone"}, "msg": "Timeo
>> ut when waiting for keystone-1:35357"}
>> ok: [10.100.198.201] => (item={u'service': u'nagios', u'ipv4_last_octet':
>> 60, u'name': u'nagios-1', u'forwarded_ports': [{u'int': 80, u'ext': 3128}],
>> u'aliases': [u'nagi
>> os']})
>> failed: [10.100.198.201] (item={u'service': u'neutron-api',
>> u'ipv4_last_octet': 70, u'name': u'neutron-api-1', u'forwarded_ports':
>> [{u'int': 9696, u'ext': 9696}], u'alia
>> ses': [u'neutron-api']}) => {"elapsed": 1800, "failed": true, "item":
>> {"aliases": ["neutron-api"], "forwarded_ports": [{"ext": 9696, "int":
>> 9696}], "ipv4_last_octet": 70
>> , "name": "neutron-api-1", "service": "neutron-api"}, "msg": "Timeout
>> when waiting for neutron-api-1:9696"}
>> failed: [10.100.198.201] (item={u'service': u'nova-cloud-controller',
>> u'ipv4_last_octet': 80, u'name': u'nova-cloud-controller-1',
>> u'forwarded_ports': [{u'int': 8774, u'
>> ext': 8774}], u'aliases': [u'nova-cloud-controller']}) => {"elapsed":
>> 1800, "failed": true, "item": {"aliases": ["nova-cloud-controller"],
>> "forwarded_ports": [{"ext": 87
>> 74, "int": 8774}], "ipv4_last_octet": 80, "name":
>> "nova-cloud-controller-1", "service": "nova-cloud-controller"}, "msg": "Timeout
>> when waiting for nova-cloud-controller-
>> 1:8774"}
>> failed: [10.100.198.201] (item={u'service': u'openstack-dashboard',
>> u'ipv4_last_octet': 90, u'name': u'openstack-dashboard-1',
>> u'forwarded_ports': [{u'int': 80, u'ext':
>> 8080}], u'aliases': [u'openstack-dashboard']}) => {"elapsed": 1800,
>> "failed": true, "item": {"aliases": ["openstack-dashboard"],
>> "forwarded_ports": [{"ext": 8080, "int":
>> 80}], "ipv4_last_octet": 90, "name": "openstack-dashboard-1", "service":
>> "openstack-dashboard"}, "msg": "Timeout when waiting for
>> openstack-dashboard-1:80"}
>> to retry, use: --limit @/cord/build/platform-install/
>> cord-deploy-openstack.retry
>>
>>
>> I tried to reset juju and lxc, but then it stuck at "Obtain Juju Facts
>> for creating machines"
>> sudo juju destroy-environment manual
>> sudo lxc delete $(sudo lxc list | grep ^\| | grep NAME -v | cut -d' ' -f
>> 2) --force
>>
>>
>>
>> --
>> Juju mailing list
>> Juju at lists.ubuntu.com
>> Modify settings or unsubscribe at: https://lists.ubuntu.com/
>> mailman/listinfo/juju
>>
>>
>>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.ubuntu.com/archives/juju/attachments/20170104/302c214b/attachment.html>
More information about the Juju
mailing list