1. ONLINE 89ccb666fba84f48bffd99f68cc37d9a (/dev/oracleasm/disks/CRS) [CRS]
Located 1 voting disk(s).
bash: asmcd: command not found...
State Type Rebal Sector Logical_Sector Block AU Total_MB Free_MB Req_mir_free_MB Usable_file_MB Offline_disks Voting_files Name
MOUNTED EXTERN N 512 512 4096 4194304 43004 8920 0 8920 0 Y CRS/
MOUNTED EXTERN N 512 512 4096 4194304 20476 20300 0 20300 0 N DATA/
MOUNTED EXTERN N 512 512 4096 4194304 20476 15592 0 15592 0 N FRA/
select a.name DiskGroup,b.name DiskName, b.total_mb, (b.total_mb-b.free_mb) Used_MB, b.free_mb,b.path,b.header_status from v$asm_disk b, v$asm_diskgroup a where a.group_number (+) =b.group_number order by b.group_number,b.name;SQL> SQL> SQL> SQL> SQL> SQL> SQL> SQL>
---------- ------------ ---------- ---------- ---------- ------------------------------ ------------
select DISK_NUMBER,MOUNT_STATUS,HEADER_STATUS,MODE_STATUS,STATE,PATH FROM V$ASM_DISK;SQL> SQL> SQL> SQL>
SQL> create diskgroup Backup external redundancy disk '/dev/oracleasm/disks/BACKUP' name backup;
Diskgroup created.
SQL> select DISK_NUMBER,MOUNT_STATUS,HEADER_STATUS,MODE_STATUS,STATE,PATH FROM V$ASM_DISK;
SQL> select a.name DiskGroup,b.name DiskName, b.total_mb, (b.total_mb-b.free_mb) Used_MB, b.free_mb,b.path,b.header_status from v$asm_disk b, v$asm_diskgroup a where a.group_number (+) =b.group_number order by b.group_number,b.name;
---------- ------------ ---------- ---------- ---------- ------------------------------ ------------
[root@srv2 ~]# crsctl stop crs -f
CRS-2791: Starting shutdown of Oracle High Availability Services-managed resources on 'srv2'
CRS-2673: Attempting to stop 'ora.crsd' on 'srv2'
CRS-2790: Starting shutdown of Cluster Ready Services-managed resources on server 'srv2'
CRS-2673: Attempting to stop 'ora.chad' on 'srv2'
CRS-2673: Attempting to stop 'ora.rac.db' on 'srv2'
CRS-2677: Stop of 'ora.rac.db' on 'srv2' succeeded
CRS-2673: Attempting to stop 'ora.DATA.dg' on 'srv2'
CRS-2673: Attempting to stop 'ora.CRS.dg' on 'srv2'
CRS-2673: Attempting to stop 'ora.FRA.dg' on 'srv2'
CRS-2673: Attempting to stop 'ora.LISTENER.lsnr' on 'srv2'
CRS-2673: Attempting to stop 'ora.LISTENER_SCAN1.lsnr' on 'srv2'
CRS-2677: Stop of 'ora.FRA.dg' on 'srv2' succeeded
CRS-2677: Stop of 'ora.DATA.dg' on 'srv2' succeeded
CRS-2677: Stop of 'ora.CRS.dg' on 'srv2' succeeded
CRS-2673: Attempting to stop 'ora.asm' on 'srv2'
CRS-2677: Stop of 'ora.LISTENER.lsnr' on 'srv2' succeeded
CRS-2673: Attempting to stop 'ora.srv2.vip' on 'srv2'
CRS-2677: Stop of 'ora.LISTENER_SCAN1.lsnr' on 'srv2' succeeded
CRS-2673: Attempting to stop 'ora.scan1.vip' on 'srv2'
CRS-2677: Stop of 'ora.asm' on 'srv2' succeeded
CRS-2673: Attempting to stop 'ora.ASMNET1LSNR_ASM.lsnr' on 'srv2'
CRS-2677: Stop of 'ora.scan1.vip' on 'srv2' succeeded
CRS-2677: Stop of 'ora.srv2.vip' on 'srv2' succeeded
CRS-2677: Stop of 'ora.chad' on 'srv2' succeeded
CRS-2677: Stop of 'ora.ASMNET1LSNR_ASM.lsnr' on 'srv2' succeeded
CRS-2672: Attempting to start 'ora.scan1.vip' on 'srv1'
CRS-2672: Attempting to start 'ora.srv2.vip' on 'srv1'
CRS-2676: Start of 'ora.srv2.vip' on 'srv1' succeeded
CRS-2676: Start of 'ora.scan1.vip' on 'srv1' succeeded
CRS-2672: Attempting to start 'ora.LISTENER_SCAN1.lsnr' on 'srv1'
CRS-2676: Start of 'ora.LISTENER_SCAN1.lsnr' on 'srv1' succeeded
CRS-2673: Attempting to stop 'ora.ons' on 'srv2'
CRS-2677: Stop of 'ora.ons' on 'srv2' succeeded
CRS-2673: Attempting to stop 'ora.net1.network' on 'srv2'
CRS-2677: Stop of 'ora.net1.network' on 'srv2' succeeded
CRS-2792: Shutdown of Cluster Ready Services-managed resources on 'srv2' has completed
CRS-2677: Stop of 'ora.crsd' on 'srv2' succeeded
CRS-2673: Attempting to stop 'ora.storage' on 'srv2'
CRS-2673: Attempting to stop 'ora.crf' on 'srv2'
CRS-2673: Attempting to stop 'ora.gpnpd' on 'srv2'
CRS-2673: Attempting to stop 'ora.mdnsd' on 'srv2'
CRS-2677: Stop of 'ora.crf' on 'srv2' succeeded
CRS-2677: Stop of 'ora.gpnpd' on 'srv2' succeeded
CRS-2677: Stop of 'ora.storage' on 'srv2' succeeded
CRS-2673: Attempting to stop 'ora.asm' on 'srv2'
CRS-2677: Stop of 'ora.mdnsd' on 'srv2' succeeded
CRS-2677: Stop of 'ora.asm' on 'srv2' succeeded
CRS-2673: Attempting to stop 'ora.cluster_interconnect.haip' on 'srv2'
CRS-2677: Stop of 'ora.cluster_interconnect.haip' on 'srv2' succeeded
CRS-2673: Attempting to stop 'ora.ctssd' on 'srv2'
CRS-2673: Attempting to stop 'ora.evmd' on 'srv2'
CRS-2677: Stop of 'ora.ctssd' on 'srv2' succeeded
CRS-2677: Stop of 'ora.evmd' on 'srv2' succeeded
CRS-2673: Attempting to stop 'ora.cssd' on 'srv2'
CRS-2677: Stop of 'ora.cssd' on 'srv2' succeeded
CRS-2673: Attempting to stop 'ora.gipcd' on 'srv2'
CRS-2677: Stop of 'ora.gipcd' on 'srv2' succeeded
CRS-2793: Shutdown of Oracle High Availability Services-managed resources on 'srv2' has completed
CRS-4133: Oracle High Availability Services has been stopped.
- Disable auto-start CRS service in case for OS reboot
[root@srv2 ~]# crsctl disable crs
CRS-4621: Oracle High Availability Services autostart is disabled.
- Start crs service with exclusive mode with no crs services
Usage:
crsctl start crs [-excl [-nocrs | -cssonly]] | [-wait | -waithas | -nowait] | [-noautostart]
Start OHAS on this server
where
-excl Start Oracle Clusterware in exclusive mode
-nocrs Start Oracle Clusterware in exclusive mode without starting CRS
-nowait Do not wait for OHAS to start
-wait Wait until startup is complete and display all progress and status messages
-waithas Wait until startup is complete and display OHASD progress and status messages
-cssonly Start only CSS
-noautostart Start only OHAS
[root@srv2 ~]# crsctl start crs -excl -nocrs
CRS-4123: Oracle High Availability Services has been started.
CRS-2672: Attempting to start 'ora.evmd' on 'srv2'
CRS-2672: Attempting to start 'ora.mdnsd' on 'srv2'
CRS-2676: Start of 'ora.mdnsd' on 'srv2' succeeded
CRS-2676: Start of 'ora.evmd' on 'srv2' succeeded
CRS-2672: Attempting to start 'ora.gpnpd' on 'srv2'
CRS-2676: Start of 'ora.gpnpd' on 'srv2' succeeded
CRS-2672: Attempting to start 'ora.cssdmonitor' on 'srv2'
CRS-2672: Attempting to start 'ora.gipcd' on 'srv2'
CRS-2676: Start of 'ora.cssdmonitor' on 'srv2' succeeded
CRS-2676: Start of 'ora.gipcd' on 'srv2' succeeded
CRS-2672: Attempting to start 'ora.cssd' on 'srv2'
CRS-2672: Attempting to start 'ora.diskmon' on 'srv2'
CRS-2676: Start of 'ora.diskmon' on 'srv2' succeeded
CRS-2676: Start of 'ora.cssd' on 'srv2' succeeded
CRS-2672: Attempting to start 'ora.cluster_interconnect.haip' on 'srv2'
CRS-2672: Attempting to start 'ora.ctssd' on 'srv2'
CRS-2676: Start of 'ora.ctssd' on 'srv2' succeeded
CRS-2676: Start of 'ora.cluster_interconnect.haip' on 'srv2' succeeded
CRS-2672: Attempting to start 'ora.asm' on 'srv2'
CRS-2676: Start of 'ora.asm' on 'srv2' succeeded
[root@srv2 ~]# crsctl query css votedisk
## STATE File Universal Id File Name Disk group
-- ----- ----------------- --------- ---------
1. ONLINE 89ccb666fba84f48bffd99f68cc37d9a (/dev/oracleasm/disks/CRS) [CRS]
Located 1 voting disk(s).
- Replace votedisk with new diskgroup. We can place votedisk with same OCR location.
[root@srv2 ~]# crsctl replace votedisk +backup
Successful addition of voting disk 6fc2ec78a9024fc2bf02824b67e7da7c.
Successful deletion of voting disk 89ccb666fba84f48bffd99f68cc37d9a.
Successfully replaced voting disk group with +backup.
CRS-4266: Voting file(s) successfully replaced
- Stop current Cluster Synchronization Services
[root@srv2 ~]# crsctl stop crs
Unable to communicate with the Cluster Synchronization Services daemon.
- Start CRS service on all nodes
[root@srv2 ~]# crsctl start cluster -all
CRS-4639: Could not contact Oracle High Availability Services
CRS-4000: Command Start failed, or completed with errors.
[root@srv2 ~]# crsctl start crs
CRS-4123: Oracle High Availability Services has been started.
[root@srv2 ~]# crsctl query css votedisk
## STATE File Universal Id File Name Disk group
-- ----- ----------------- --------- ---------
1. ONLINE 6fc2ec78a9024fc2bf02824b67e7da7c (/dev/oracleasm/disks/BACKUP) [BACKUP]
Located 1 voting disk(s).
[root@srv2 ~]#
- Check all clusterware services
[root@srv2 ~]# crsctl status resource -t
--------------------------------------------------------------------------------
Name Target State Server State details
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.ASMNET1LSNR_ASM.lsnr
ONLINE ONLINE srv1 STABLE
ONLINE ONLINE srv2 STABLE
ora.BACKUP.dg
ONLINE ONLINE srv1 STABLE
ONLINE ONLINE srv2 STABLE
ora.CRS.dg
ONLINE ONLINE srv1 STABLE
ONLINE ONLINE srv2 STABLE
ora.DATA.dg
ONLINE ONLINE srv1 STABLE
ONLINE ONLINE srv2 STABLE
ora.FRA.dg
ONLINE ONLINE srv1 STABLE
ONLINE ONLINE srv2 STABLE
ora.LISTENER.lsnr
ONLINE ONLINE srv1 STABLE
ONLINE ONLINE srv2 STABLE
ora.chad
ONLINE ONLINE srv1 STABLE
ONLINE ONLINE srv2 STABLE
ora.net1.network
ONLINE ONLINE srv1 STABLE
ONLINE ONLINE srv2 STABLE
ora.ons
ONLINE ONLINE srv1 STABLE
ONLINE ONLINE srv2 STABLE
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.LISTENER_SCAN1.lsnr
1 ONLINE ONLINE srv1 STABLE
ora.LISTENER_SCAN2.lsnr
1 ONLINE ONLINE srv2 STABLE
ora.LISTENER_SCAN3.lsnr
1 ONLINE ONLINE srv2 STABLE
ora.MGMTLSNR
1 ONLINE ONLINE srv2 169.254.238.221 192.
168.10.2,STABLE
ora.asm
1 ONLINE ONLINE srv1 Started,STABLE
2 ONLINE ONLINE srv2 Started,STABLE
3 ONLINE OFFLINE STABLE
ora.cvu
1 ONLINE ONLINE srv2 STABLE
ora.mgmtdb
1 ONLINE ONLINE srv2 Open,STABLE
ora.qosmserver
1 ONLINE ONLINE srv2 STABLE
ora.rac.db
1 ONLINE ONLINE srv1 Open,HOME=/u01/app/o
racle/product/12.2.0
/db_1,STABLE
2 ONLINE ONLINE srv2 Open,HOME=/u01/app/o
racle/product/12.2.0
/db_1,STABLE
3 ONLINE OFFLINE STABLE
ora.scan1.vip
1 ONLINE ONLINE srv1 STABLE
ora.scan2.vip
1 ONLINE ONLINE srv2 STABLE
ora.scan3.vip
1 ONLINE ONLINE srv2 STABLE
ora.srv1.vip
1 ONLINE ONLINE srv1 STABLE
ora.srv2.vip
1 ONLINE ONLINE srv2 STABLE
ora.srv3.vip
1 ONLINE INTERMEDIATE srv2 FAILED OVER,STABLE
--------------------------------------------------------------------------------
- Now Start node 3 and check vote disk status
[root@srv2 ~]# ssh srv3
root@srv3's password:
Last login: Tue Nov 2 16:13:08 2021 from srv1.example.com
[root@srv3 ~]# . oraenv
ORACLE_SID = [root] ? +ASM3
The Oracle base has been set to /u01/app/grid
[root@srv3 ~]# crsctl query css votedisk
## STATE File Universal Id File Name Disk group
-- ----- ----------------- --------- ---------
1. ONLINE 6fc2ec78a9024fc2bf02824b67e7da7c (/dev/oracleasm/disks/BACKUP) [BACKUP]
Located 1 voting disk(s).
[root@srv3 ~]# olsnodes
srv1
srv2
srv3
- Check clusrtware status from node 3
[root@srv3 ~]# crsctl check cluster -all
**************************************************************
srv1:
CRS-4537: Cluster Ready Services is online
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online
**************************************************************
srv2:
CRS-4537: Cluster Ready Services is online
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online
**************************************************************
srv3:
CRS-4537: Cluster Ready Services is online
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online
**************************************************************
[root@srv3 ~]# crsctl status resource -t
--------------------------------------------------------------------------------
Name Target State Server State details
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.ASMNET1LSNR_ASM.lsnr
ONLINE ONLINE srv1 STABLE
ONLINE ONLINE srv2 STABLE
ONLINE ONLINE srv3 STABLE
ora.BACKUP.dg
ONLINE ONLINE srv1 STABLE
ONLINE ONLINE srv2 STABLE
ONLINE ONLINE srv3 STABLE
ora.CRS.dg
ONLINE ONLINE srv1 STABLE
ONLINE ONLINE srv2 STABLE
ONLINE ONLINE srv3 STABLE
ora.DATA.dg
ONLINE ONLINE srv1 STABLE
ONLINE ONLINE srv2 STABLE
ONLINE ONLINE srv3 STABLE
ora.FRA.dg
ONLINE ONLINE srv1 STABLE
ONLINE ONLINE srv2 STABLE
ONLINE ONLINE srv3 STABLE
ora.LISTENER.lsnr
ONLINE ONLINE srv1 STABLE
ONLINE ONLINE srv2 STABLE
ONLINE ONLINE srv3 STABLE
ora.chad
ONLINE ONLINE srv1 STABLE
ONLINE ONLINE srv2 STABLE
OFFLINE OFFLINE srv3 STABLE
ora.net1.network
ONLINE ONLINE srv1 STABLE
ONLINE ONLINE srv2 STABLE
ONLINE ONLINE srv3 STABLE
ora.ons
ONLINE ONLINE srv1 STABLE
ONLINE ONLINE srv2 STABLE
ONLINE ONLINE srv3 STABLE
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.LISTENER_SCAN1.lsnr
1 ONLINE ONLINE srv1 STABLE
ora.LISTENER_SCAN2.lsnr
1 ONLINE ONLINE srv2 STABLE
ora.LISTENER_SCAN3.lsnr
1 ONLINE ONLINE srv3 STABLE
ora.MGMTLSNR
1 ONLINE ONLINE srv3 169.254.187.26 192.1
68.10.3,STABLE
ora.asm
1 ONLINE ONLINE srv1 Started,STABLE
2 ONLINE ONLINE srv2 Started,STABLE
3 ONLINE ONLINE srv3 Started,STABLE
ora.cvu
1 ONLINE ONLINE srv3 STABLE
ora.mgmtdb
1 ONLINE ONLINE srv3 Open,STABLE
ora.qosmserver
1 ONLINE ONLINE srv3 STABLE
ora.rac.db
1 ONLINE ONLINE srv1 Open,HOME=/u01/app/o
racle/product/12.2.0
/db_1,STABLE
2 ONLINE ONLINE srv2 Open,HOME=/u01/app/o
racle/product/12.2.0
/db_1,STABLE
3 ONLINE ONLINE srv3 Open,HOME=/u01/app/o
racle/product/12.2.0
/db_1,STABLE
ora.scan1.vip
1 ONLINE ONLINE srv1 STABLE
ora.scan2.vip
1 ONLINE ONLINE srv2 STABLE
ora.scan3.vip
1 ONLINE ONLINE srv3 STABLE
ora.srv1.vip
1 ONLINE ONLINE srv1 STABLE
ora.srv2.vip
1 ONLINE ONLINE srv2 STABLE
ora.srv3.vip
1 ONLINE ONLINE srv3 STABLE
--------------------------------------------------------------------------------