2026-03-31T18:56:44.396 INFO:root:teuthology version: 1.2.4.dev37+ga59626679 2026-03-31T18:56:44.404 DEBUG:teuthology.report:Pushing job info to http://localhost:8080 2026-03-31T18:56:44.426 INFO:teuthology.run:Config: archive_path: /archive/kyr-2026-03-31_11:18:10-rados-tentacle-none-default-vps/4335 branch: tentacle description: rados/cephadm/osds/{0-distro/centos_9.stream_runc 0-nvme-loop 1-start 2-ops/repave-all} email: null first_in_suite: false flavor: default job_id: '4335' last_in_suite: false machine_type: vps name: kyr-2026-03-31_11:18:10-rados-tentacle-none-default-vps no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: tentacle ansible.cephlab: branch: main repo: https://github.com/kshtsk/ceph-cm-ansible.git skip_tags: nagios,monitoring-scripts,hostname,pubkeys,zap,sudoers,kerberos,ntp-client,resolvconf,cpan,nfs vars: logical_volumes: lv_1: scratch_dev: true size: 25%VG vg: vg_nvme lv_2: scratch_dev: true size: 25%VG vg: vg_nvme lv_3: scratch_dev: true size: 25%VG vg: vg_nvme lv_4: scratch_dev: true size: 25%VG vg: vg_nvme timezone: UTC volume_groups: vg_nvme: pvs: /dev/vdb,/dev/vdc,/dev/vdd,/dev/vde ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd mclock iops capacity threshold hdd: 49000 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - OSD_DOWN - CEPHADM_FAILED_DAEMON - but is still running - PG_DEGRADED log-only-match: - CEPHADM_ sha1: 5bb3278730741031382ca9c3dc9d221a942e06a2 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} cephadm: cephadm_binary_url: https://download.ceph.com/rpm-20.2.0/el9/noarch/cephadm install: ceph: flavor: default sha1: 5bb3278730741031382ca9c3dc9d221a942e06a2 extra_system_packages: deb: - python3-jmespath - python3-xmltodict - s3cmd rpm: - bzip2 - perl-Test-Harness - python3-jmespath - python3-xmltodict - s3cmd selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: tt-tentacle sha1: 0392f78529848ec72469e8e431875cb98d3a5fb4 owner: kyr priority: 1000 repo: https://github.com/ceph/ceph.git roles: - - host.a - client.0 - - host.b - client.1 seed: 6407 sha1: 5bb3278730741031382ca9c3dc9d221a942e06a2 sleep_before_teardown: 0 subset: 1/100000 suite: rados suite_branch: tt-tentacle suite_path: /home/teuthos/src/github.com_kshtsk_ceph_0392f78529848ec72469e8e431875cb98d3a5fb4/qa suite_relpath: qa suite_repo: https://github.com/kshtsk/ceph.git suite_sha1: 0392f78529848ec72469e8e431875cb98d3a5fb4 targets: vm00.local: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOm5qVJmuB23caa4P/7MJNRRMEUP1BAKwHONlfPS5g5QSKqGcBA5q6S1ZNmG4uSq/QBXHzqUt4iAz7U1IpXZ9Jk= vm04.local: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBF6TLbs2YUGK1QuCT9hDMTJsCX7iQerga1fgyQAq5d5SHrwecOIdh0rsMT6g/aYTH7vafYlfuI5B9YRk+He7ADk= tasks: - pexec: all: - sudo dnf remove nvme-cli -y - sudo dnf install runc nvmetcli nvme-cli -y - sudo sed -i 's/^#runtime = "crun"/runtime = "runc"/g' /usr/share/containers/containers.conf - sudo sed -i 's/runtime = "crun"/#runtime = "crun"/g' /usr/share/containers/containers.conf - nvme_loop: null - cephadm: roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - ceph orch ls | grep '^osd.all-available-devices ' - cephadm.shell: host.a: - "set -e\nset -x\nceph orch ps\nceph orch device ls\nceph osd tree\nfor osd in\ \ `ceph osd ls` ; do\n ceph orch osd rm $osd --force --zap --replace\ndone\n\ while ceph orch osd rm ls | wc | grep ^1 ; do sleep 10 ; done\n" teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: uv2 teuthology_repo: https://github.com/kshtsk/teuthology teuthology_sha1: a59626679648f962bca99d20d35578f2998c8f37 timestamp: 2026-03-31_11:18:10 tube: vps user: kyr verbose: false worker_log: /home/teuthos/.teuthology/dispatcher/dispatcher.vps.282426 2026-03-31T18:56:44.426 INFO:teuthology.run:suite_path is set to /home/teuthos/src/github.com_kshtsk_ceph_0392f78529848ec72469e8e431875cb98d3a5fb4/qa; will attempt to use it 2026-03-31T18:56:44.426 INFO:teuthology.run:Found tasks at /home/teuthos/src/github.com_kshtsk_ceph_0392f78529848ec72469e8e431875cb98d3a5fb4/qa/tasks 2026-03-31T18:56:44.426 INFO:teuthology.run_tasks:Running task internal.check_packages... 2026-03-31T18:56:44.426 INFO:teuthology.task.internal:Checking packages... 2026-03-31T18:56:44.427 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '5bb3278730741031382ca9c3dc9d221a942e06a2' 2026-03-31T18:56:44.427 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2026-03-31T18:56:44.427 INFO:teuthology.packaging:ref: None 2026-03-31T18:56:44.427 INFO:teuthology.packaging:tag: None 2026-03-31T18:56:44.427 INFO:teuthology.packaging:branch: tentacle 2026-03-31T18:56:44.427 INFO:teuthology.packaging:sha1: 5bb3278730741031382ca9c3dc9d221a942e06a2 2026-03-31T18:56:44.427 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=tentacle 2026-03-31T18:56:45.213 INFO:teuthology.task.internal:Found packages for ceph version 20.2.0-721.g5bb32787 2026-03-31T18:56:45.213 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2026-03-31T18:56:45.214 INFO:teuthology.task.internal:no buildpackages task found 2026-03-31T18:56:45.214 INFO:teuthology.run_tasks:Running task internal.save_config... 2026-03-31T18:56:45.214 INFO:teuthology.task.internal:Saving configuration 2026-03-31T18:56:45.219 INFO:teuthology.run_tasks:Running task internal.check_lock... 2026-03-31T18:56:45.220 INFO:teuthology.task.internal.check_lock:Checking locks... 2026-03-31T18:56:45.226 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'vm00.local', 'description': '/archive/kyr-2026-03-31_11:18:10-rados-tentacle-none-default-vps/4335', 'up': True, 'machine_type': 'vps', 'is_vm': True, 'vm_host': {'name': 'localhost', 'description': None, 'up': True, 'machine_type': 'libvirt', 'is_vm': False, 'vm_host': None, 'os_type': None, 'os_version': None, 'arch': None, 'locked': True, 'locked_since': None, 'locked_by': None, 'mac_address': None, 'ssh_pub_key': None}, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-03-31 18:55:04.647482', 'locked_by': 'kyr', 'mac_address': '52:55:00:00:00:00', 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOm5qVJmuB23caa4P/7MJNRRMEUP1BAKwHONlfPS5g5QSKqGcBA5q6S1ZNmG4uSq/QBXHzqUt4iAz7U1IpXZ9Jk='} 2026-03-31T18:56:45.231 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'vm04.local', 'description': '/archive/kyr-2026-03-31_11:18:10-rados-tentacle-none-default-vps/4335', 'up': True, 'machine_type': 'vps', 'is_vm': True, 'vm_host': {'name': 'localhost', 'description': None, 'up': True, 'machine_type': 'libvirt', 'is_vm': False, 'vm_host': None, 'os_type': None, 'os_version': None, 'arch': None, 'locked': True, 'locked_since': None, 'locked_by': None, 'mac_address': None, 'ssh_pub_key': None}, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-03-31 18:55:04.647003', 'locked_by': 'kyr', 'mac_address': '52:55:00:00:00:04', 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBF6TLbs2YUGK1QuCT9hDMTJsCX7iQerga1fgyQAq5d5SHrwecOIdh0rsMT6g/aYTH7vafYlfuI5B9YRk+He7ADk='} 2026-03-31T18:56:45.231 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2026-03-31T18:56:45.232 INFO:teuthology.task.internal:roles: ubuntu@vm00.local - ['host.a', 'client.0'] 2026-03-31T18:56:45.232 INFO:teuthology.task.internal:roles: ubuntu@vm04.local - ['host.b', 'client.1'] 2026-03-31T18:56:45.232 INFO:teuthology.run_tasks:Running task console_log... 2026-03-31T18:56:45.237 DEBUG:teuthology.task.console_log:vm00 does not support IPMI; excluding 2026-03-31T18:56:45.242 DEBUG:teuthology.task.console_log:vm04 does not support IPMI; excluding 2026-03-31T18:56:45.242 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fe6fa06e290>, signals=[15]) 2026-03-31T18:56:45.242 INFO:teuthology.run_tasks:Running task internal.connect... 2026-03-31T18:56:45.243 INFO:teuthology.task.internal:Opening connections... 2026-03-31T18:56:45.243 DEBUG:teuthology.task.internal:connecting to ubuntu@vm00.local 2026-03-31T18:56:45.244 DEBUG:teuthology.orchestra.connection:{'hostname': 'vm00.local', 'username': 'ubuntu', 'timeout': 60} 2026-03-31T18:56:45.301 DEBUG:teuthology.task.internal:connecting to ubuntu@vm04.local 2026-03-31T18:56:45.301 DEBUG:teuthology.orchestra.connection:{'hostname': 'vm04.local', 'username': 'ubuntu', 'timeout': 60} 2026-03-31T18:56:45.360 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2026-03-31T18:56:45.362 DEBUG:teuthology.orchestra.run.vm00:> uname -m 2026-03-31T18:56:45.398 INFO:teuthology.orchestra.run.vm00.stdout:x86_64 2026-03-31T18:56:45.399 DEBUG:teuthology.orchestra.run.vm00:> cat /etc/os-release 2026-03-31T18:56:45.456 INFO:teuthology.orchestra.run.vm00.stdout:NAME="CentOS Stream" 2026-03-31T18:56:45.456 INFO:teuthology.orchestra.run.vm00.stdout:VERSION="9" 2026-03-31T18:56:45.456 INFO:teuthology.orchestra.run.vm00.stdout:ID="centos" 2026-03-31T18:56:45.456 INFO:teuthology.orchestra.run.vm00.stdout:ID_LIKE="rhel fedora" 2026-03-31T18:56:45.456 INFO:teuthology.orchestra.run.vm00.stdout:VERSION_ID="9" 2026-03-31T18:56:45.456 INFO:teuthology.orchestra.run.vm00.stdout:PLATFORM_ID="platform:el9" 2026-03-31T18:56:45.456 INFO:teuthology.orchestra.run.vm00.stdout:PRETTY_NAME="CentOS Stream 9" 2026-03-31T18:56:45.456 INFO:teuthology.orchestra.run.vm00.stdout:ANSI_COLOR="0;31" 2026-03-31T18:56:45.456 INFO:teuthology.orchestra.run.vm00.stdout:LOGO="fedora-logo-icon" 2026-03-31T18:56:45.456 INFO:teuthology.orchestra.run.vm00.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2026-03-31T18:56:45.456 INFO:teuthology.orchestra.run.vm00.stdout:HOME_URL="https://centos.org/" 2026-03-31T18:56:45.456 INFO:teuthology.orchestra.run.vm00.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2026-03-31T18:56:45.456 INFO:teuthology.orchestra.run.vm00.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2026-03-31T18:56:45.456 INFO:teuthology.orchestra.run.vm00.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2026-03-31T18:56:45.457 INFO:teuthology.lock.ops:Updating vm00.local on lock server 2026-03-31T18:56:45.460 DEBUG:teuthology.orchestra.run.vm04:> uname -m 2026-03-31T18:56:45.477 INFO:teuthology.orchestra.run.vm04.stdout:x86_64 2026-03-31T18:56:45.477 DEBUG:teuthology.orchestra.run.vm04:> cat /etc/os-release 2026-03-31T18:56:45.533 INFO:teuthology.orchestra.run.vm04.stdout:NAME="CentOS Stream" 2026-03-31T18:56:45.533 INFO:teuthology.orchestra.run.vm04.stdout:VERSION="9" 2026-03-31T18:56:45.533 INFO:teuthology.orchestra.run.vm04.stdout:ID="centos" 2026-03-31T18:56:45.533 INFO:teuthology.orchestra.run.vm04.stdout:ID_LIKE="rhel fedora" 2026-03-31T18:56:45.533 INFO:teuthology.orchestra.run.vm04.stdout:VERSION_ID="9" 2026-03-31T18:56:45.533 INFO:teuthology.orchestra.run.vm04.stdout:PLATFORM_ID="platform:el9" 2026-03-31T18:56:45.533 INFO:teuthology.orchestra.run.vm04.stdout:PRETTY_NAME="CentOS Stream 9" 2026-03-31T18:56:45.533 INFO:teuthology.orchestra.run.vm04.stdout:ANSI_COLOR="0;31" 2026-03-31T18:56:45.533 INFO:teuthology.orchestra.run.vm04.stdout:LOGO="fedora-logo-icon" 2026-03-31T18:56:45.533 INFO:teuthology.orchestra.run.vm04.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2026-03-31T18:56:45.533 INFO:teuthology.orchestra.run.vm04.stdout:HOME_URL="https://centos.org/" 2026-03-31T18:56:45.533 INFO:teuthology.orchestra.run.vm04.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2026-03-31T18:56:45.533 INFO:teuthology.orchestra.run.vm04.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2026-03-31T18:56:45.533 INFO:teuthology.orchestra.run.vm04.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2026-03-31T18:56:45.534 INFO:teuthology.lock.ops:Updating vm04.local on lock server 2026-03-31T18:56:45.537 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2026-03-31T18:56:45.539 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2026-03-31T18:56:45.540 INFO:teuthology.task.internal:Checking for old test directory... 2026-03-31T18:56:45.541 DEBUG:teuthology.orchestra.run.vm00:> test '!' -e /home/ubuntu/cephtest 2026-03-31T18:56:45.542 DEBUG:teuthology.orchestra.run.vm04:> test '!' -e /home/ubuntu/cephtest 2026-03-31T18:56:45.589 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2026-03-31T18:56:45.590 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2026-03-31T18:56:45.590 DEBUG:teuthology.orchestra.run.vm00:> test -z $(ls -A /var/lib/ceph) 2026-03-31T18:56:45.597 DEBUG:teuthology.orchestra.run.vm04:> test -z $(ls -A /var/lib/ceph) 2026-03-31T18:56:45.612 INFO:teuthology.orchestra.run.vm00.stderr:ls: cannot access '/var/lib/ceph': No such file or directory 2026-03-31T18:56:45.643 INFO:teuthology.orchestra.run.vm04.stderr:ls: cannot access '/var/lib/ceph': No such file or directory 2026-03-31T18:56:45.643 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2026-03-31T18:56:45.651 DEBUG:teuthology.orchestra.run.vm00:> test -e /ceph-qa-ready 2026-03-31T18:56:45.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-31T18:56:45.867 DEBUG:teuthology.orchestra.run.vm04:> test -e /ceph-qa-ready 2026-03-31T18:56:45.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-31T18:56:46.066 INFO:teuthology.run_tasks:Running task internal.base... 2026-03-31T18:56:46.067 INFO:teuthology.task.internal:Creating test directory... 2026-03-31T18:56:46.067 DEBUG:teuthology.orchestra.run.vm00:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-03-31T18:56:46.069 DEBUG:teuthology.orchestra.run.vm04:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-03-31T18:56:46.084 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2026-03-31T18:56:46.085 INFO:teuthology.run_tasks:Running task internal.archive... 2026-03-31T18:56:46.086 INFO:teuthology.task.internal:Creating archive directory... 2026-03-31T18:56:46.086 DEBUG:teuthology.orchestra.run.vm00:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-03-31T18:56:46.125 DEBUG:teuthology.orchestra.run.vm04:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-03-31T18:56:46.141 INFO:teuthology.run_tasks:Running task internal.coredump... 2026-03-31T18:56:46.143 INFO:teuthology.task.internal:Enabling coredump saving... 2026-03-31T18:56:46.143 DEBUG:teuthology.orchestra.run.vm00:> test -f /run/.containerenv -o -f /.dockerenv 2026-03-31T18:56:46.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-31T18:56:46.196 DEBUG:teuthology.orchestra.run.vm04:> test -f /run/.containerenv -o -f /.dockerenv 2026-03-31T18:56:46.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-03-31T18:56:46.210 DEBUG:teuthology.orchestra.run.vm00:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-03-31T18:56:46.237 DEBUG:teuthology.orchestra.run.vm04:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-03-31T18:56:46.260 INFO:teuthology.orchestra.run.vm00.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-03-31T18:56:46.268 INFO:teuthology.orchestra.run.vm00.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-03-31T18:56:46.274 INFO:teuthology.orchestra.run.vm04.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-03-31T18:56:46.283 INFO:teuthology.orchestra.run.vm04.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-03-31T18:56:46.285 INFO:teuthology.run_tasks:Running task internal.sudo... 2026-03-31T18:56:46.303 INFO:teuthology.task.internal:Configuring sudo... 2026-03-31T18:56:46.303 DEBUG:teuthology.orchestra.run.vm00:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-03-31T18:56:46.311 DEBUG:teuthology.orchestra.run.vm04:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-03-31T18:56:46.348 INFO:teuthology.run_tasks:Running task internal.syslog... 2026-03-31T18:56:46.351 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2026-03-31T18:56:46.351 DEBUG:teuthology.orchestra.run.vm00:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-03-31T18:56:46.375 DEBUG:teuthology.orchestra.run.vm04:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-03-31T18:56:46.404 DEBUG:teuthology.orchestra.run.vm00:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-03-31T18:56:46.452 DEBUG:teuthology.orchestra.run.vm00:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-03-31T18:56:46.509 DEBUG:teuthology.orchestra.run.vm00:> set -ex 2026-03-31T18:56:46.509 DEBUG:teuthology.orchestra.run.vm00:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-03-31T18:56:46.569 DEBUG:teuthology.orchestra.run.vm04:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-03-31T18:56:46.590 DEBUG:teuthology.orchestra.run.vm04:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-03-31T20:56:49.922 DEBUG:teuthology.exit:Got signal 15; running 1 handler... 2026-03-31T20:56:49.923 DEBUG:teuthology.exit:Finished running handlers