rhel7.6平台Oracle19c数据库集群部署文档3-ORACLE 19C补丁升级至RU6 步骤

一.背景介绍

​ Oracle Database 19c会作为Oracle长期支持的版本.所以在初始安装19.3版本后，需要应用最近补丁集（2020年1月推荐补丁集RU6），升级软件至19.6版本。

二、安装所需介质

	oracle_12.2.0.3_for_LINUX_64_202001_RU6.zip
	oracle_12.2.0.3_for_LINUX_64_2020年1月推荐补丁集RU6.zip
	
	sftp 到/home/backupfile/orapatch
	# ls -l /home/backupfile/orapatch
	-rwxrwxrwx 1 grid oinstall 2160976478 Mar 13 08:49 p30501910_190000_Linux-x86-64.zip
		--主补丁包
	-rwxrwxrwx 1 grid oinstall     244262 Mar 12 21:09 p30396974_196000DBRU_Linux-x86-64.zip
		--30396974补丁包
	-rwxrwxrwx 1 grid oinstall    2188984 Mar 12 21:10 p27661222_196000DBRU_Linux-x86-64.zip
	-rwxrwxrwx 1 grid oinstall      47898 Mar 20 15:53 p28837979_196000DBRU_Linux-x86-64.zip
	-rwxrwxrwx 1 grid oinstall      39014 Mar 12 21:07 p28910498_196000DBRU_Linux-x86-64.zip
	-rwxrwxrwx 1 grid oinstall     503101 Mar 12 21:14 p30486436_196000DBRU_Linux-x86-64.zip
		--零星补丁
	-rw-r--r-- 1 root root     1986870968 Aug  7 12:04 p29699097_190000_Linux-x86-64.zip
		--解决第二节点安装bug时的主补丁依赖
		
		
	-rwxrwxrwx 1 grid oinstall  115653541 Mar 13 14:03 p6880880_190000_Linux-x86-64.zip   
		--OPatch工具
	-rw-r--r-- 1 root root         245041 Aug  6 13:28 p30396974_194000DBRU_Linux-x86-64.zip  
		--FOR19.4的，不需要
		

三、操作步骤

3.1 备份现有oracle和grid目录

	--需50G左右
	【备份之前先停止集群】
	使用root用户,分别在各个节点执行
	su - root
	#  source /home/db/grid/.bash_profile; crsctl stop cluster
	# /source /home/db/grid/.bash_profile; crsctl stop crs
	
	【backup oracle and grid】
	# ls /home/db/grid/product/12.2.0/rdbms/audit | wc  -l
	# rsync --delete-before -d /home/backupfile/tmp/blank/ /home/db/grid/product/12.2.0/rdbms/audit/
	# cd /home/db;tar cvf /home/backupfile/ora_backup/oracle.tar oracle
	# cd /home/db;tar cvf /home/backupfile/ora_backup/grid.tar grid
	
	mkdir -p /home/backupfile/orapatch
	chown -R grid:oinstall /home/backupfile/orapatch/
	chmod -R 777 /home/backupfile/orapatch/

3.2 解压补丁并更新OPatch

3. 解压补丁并更新OPatch
	路径：oracle/oracle_12.2.0.3_for_LINUX_64_202001_RU6.zip
	

	更新OPatch
	# su - grid
	$ cd $ORACLE_HOME
	$ ./OPatch/opatch version
		12.2.0.1.17
		
	# su - oracle
	$ cd $ORACLE_HOME
	$ ./OPatch/opatch version
	
	
	--- 解压补丁包
	# su - root
	# cd /home/backupfile/orapatch; unzip p30501910_190000_Linux-x86-64.zip
	# chmod 777 30501910/ -R
	

3.3 应用主补丁集(30501910)

​ 【节点1打完，再在节点2上重复一编操作，打节点2】

节点1

	--检查是否启动crs。crs要启动，否则报错
	# source /home/db/grid/.bash_profile  && crsctl start crs
	# crsctl query crs softwarepatch
		Oracle Clusterware patch level on node bjdb19c0 is [724960844].


	# su - grid
	$ opatch lspatches
		29585399;OCW RELEASE UPDATE 19.3.0.0.0 (29585399)
		29517247;ACFS RELEASE UPDATE 19.3.0.0.0 (29517247)
		29517242;Database Release Update : 19.3.0.0.190416 (29517242)
		29401763;TOMCAT RELEASE UPDATE 19.0.0.0.0 (29401763)
			--版本都为19.3
	# su - oracle
	$ opatch lspatches
		29585399;OCW RELEASE UPDATE 19.3.0.0.0 (29585399)
		29517242;Database Release Update : 19.3.0.0.190416 (29517242)
			--版本都为19.3
	
	【开始打补丁】
	# su - root
	# export PATH=$PATH:/home/db/grid/product/12.2.0/OPatch

	
	--- 安装前分析 
	# opatchauto apply /home/backupfile/orapatch/30501910/  -analyze

	--指定gi home，升级gi。约10分钟
	# opatchauto apply /home/backupfile/orapatch/30501910 -oh /home/db/grid/product/12.2.0
	
	--指定oracle home,升级db。约4分钟
	# exit 
		--必须退出再登录一次。	
	# su - root
	# export PATH=$PATH:/home/db/oracle/product/12.2.0/OPatch
	# opatchauto apply -oh /home/db/oracle/product/12.2.0 /home/backupfile/orapatch/30501910 
	
	
	# su - grid 
	$ opatch lspatches
		30655595;TOMCAT RELEASE UPDATE 19.0.0.0.0 (30655595)
		30557433;Database Release Update : 19.6.0.0.200114 (30557433)
		30489632;ACFS RELEASE UPDATE 19.6.0.0.0 (30489632)
		30489227;OCW RELEASE UPDATE 19.6.0.0.0 (30489227)		
		--版本都升到19.6了
	# su - oracle 
	$ opatch lspatches
		30557433;Database Release Update : 19.6.0.0.200114 (30557433)
		30489227;OCW RELEASE UPDATE 19.6.0.0.0 (30489227)
		--都升到19.6了	
		

	--未做dbca，没有数据库实例，以下忽略不执行
	su - oracle
	sqlplus / as sysdba
	alter pluggable database all open;
	exit
	cd $ORACLE_HOME/OPatch
	./datapatch -verbose

3.4 应用crs和db补丁(30396974)

其次打crs和db的补丁(30396974)

	#su - root
	# cd /home/backupfile/orapatch/ ; unzip p30396974_194000DBRU_Linux-x86-64.zip
	# cd /home/backupfile/orapatch/ ;chmod 777 30396974/ -R
	
	
	oracle用户停db库：--未dbca，可以不做
	srvctl stop instance -d cen00019 -n w412pc11
		
	
	root用户执行，预处理：
	# exit
	# su - root
	# /home/db/grid/product/12.2.0/crs/install/rootcrs.sh -prepatch
		-- 4分钟左右（要关闭crs服务，及卸载HOME目录）
		
	su - grid
	$ /home/db/grid/product/12.2.0/OPatch/opatch apply -oh /home/db/grid/product/12.2.0 -local /home/backupfile/orapatch/30396974
		-- 回答两个y
		
	su - oracle
	$ /home/db/oracle/product/12.2.0/OPatch/opatch apply -oh /home/db/oracle/product/12.2.0 -local /home/backupfile/orapatch/30396974
		-- 回答两个y	
	
	root用户执行，后处理：
	# exit
	su - root
	# /home/db/grid/product/12.2.0/crs/install/rootcrs.sh -postpatch
		-- 10分钟左右

3.5 再打 DB补丁（28910498，2883797，2766122，30486436）

再打 DB补丁28910498，2883797，2766122，30486436

	进入对应补丁目录并使用root执行

	su - root
	
	# cd /home/backupfile/orapatch/
	# unzip p28910498_196000DBRU_Linux-x86-64.zip
	# unzip p28837979_196000DBRU_Linux-x86-64.zip
	# unzip p27661222_196000DBRU_Linux-x86-64.zip
	# unzip p30486436_196000DBRU_Linux-x86-64.zip
	# chmod 777 * -R
	
	su - oracle
	# cd /home/backupfile/orapatch/
	# cd 28910498 && /home/db/oracle/product/12.2.0/OPatch/opatch apply
		-- 回答两个y	
	# cd ../28837979  && /home/db/oracle/product/12.2.0/OPatch/opatch apply
		-- 回答两个y	
	# cd ../27661222 && /home/db/oracle/product/12.2.0/OPatch/opatch apply
		-- 回答两个y	
	# cd ../30486436 && /home/db/oracle/product/12.2.0/OPatch/opatch apply
		-- 回答两个y	


		

3.6 验证补丁结果

完成补丁后。

	CLUSTER补丁如下：
	su - grid
	# opatch lspatches
		30396974;REBALANCE ABORTING WITH ORA-15424
		30655595;TOMCAT RELEASE UPDATE 19.0.0.0.0 (30655595)
		30557433;Database Release Update : 19.6.0.0.200114 (30557433)
		30489632;ACFS RELEASE UPDATE 19.6.0.0.0 (30489632)
		30489227;OCW RELEASE UPDATE 19.6.0.0.0 (30489227)



	RDBMS补丁如下：
	su - oracle
	# opatch lspatches
		30486436;19.4 AIX ORA-00600--[KJCSMPAV 1], [0X7000105CC424AF0], [4], [0], [211], [220]
		27661222;WRONG RESULTS WITH PARTITION PRUNING--COMPARING CHAR COLUMNS OF DIFFERENT LENGTH
		28837979;WRONG RESULT PARALLEL EXEC WITH ROWNUM AND ORA_HASH
		28910498;WRONG RESULT WITH JOIN TABLES WITH CHAR COLUMNS OF DIFFERENT LENGTH
		30396974;REBALANCE ABORTING WITH ORA-15424
		30557433;Database Release Update : 19.6.0.0.200114 (30557433)
		30489227;OCW RELEASE UPDATE 19.6.0.0.0 (30489227)
	

3.7 在第二节点，进行补丁安装

	
在第二节点，进行补丁安装。步骤和节点1一致。但是
	注意：在安装主补丁时，升级gi执行《# opatchauto apply /home/backupfile/orapatch/30501910 -oh /home/db/grid/product/12.2.0》中，会出现异常。
	首先，会报【 Unable to create patchObject】错，可按 5项故障处理解决
	随后，会提示【requires prerequisite patch(es) [29834717]】需要中间补丁，可按第6项故障处理解决

四、故障处理

故障处理1【Configure and start the Clusterware】

以下为故障处理：

故障处理1【Configure and start the Clusterware】

	Clusterware is either not running or not configured. You have the following 2 options:
	 1. Configure and start the Clusterware on this node and re-run the tool
	 2. Run the tool with '-oh ' to first patch the Grid Home, then invoke tool with '-database ' or '-oh '
	to patch the RAC home


	启动本地节点的crs集群服务。

故障处理2【回退grid和oracle目录备份】

故障处理2【回退grid和oracle目录备份】

	升级中途失败
	升级主补丁，漏了在oracle目录上执行。后面补丁提示依赖，必须回退
	
	--关闭RAC随系统启动自动
	# systemctl status oracle-ohasd.service
	# systemctl disable oracle-ohasd.service
	# reboot


	# cd /home/db;tar xvf /home/backupfile/ora_backup/oracle.tar
	# cd /home/db;tar xvf /home/backupfile/ora_backup/grid.tar
	
	恢复RAC随系统启动自动
	# systemctl enable oracle-ohasd.service

	# reboot
	# systemctl status oracle-ohasd.service
	





	

故障处理3【ORA-15139: patch level of ASM instances differ in cluster】

	
	去升级另外一个节点，成功后在回来执行# opatchauto resume

故障处理4【oui-patch.xml (Permission denied)】

	Patch: /home/backupfile/orapatch/30501910/30489227
	Log: /home/db/oracle/product/12.2.0/cfgtoollogs/opatchauto/core/opatch/opatch2020-08-06_21-35-34PM_1.log
	Reason: Failed during Patching: oracle.opatch.opatchsdk.OPatchException: ApplySession failed in system modification phase... 'ApplySession::apply failed: java.io.IOException: oracle.sysman.oui.patch.PatchException: java.io.FileNotFoundException: /home/db/grid/oraInventory/ContentsXML/oui-patch.xml (Permission denied)' 

--处理
	# chmod 660 /home/db/grid/oraInventory/ContentsXML/oui-patch.xml
	# opatchauto resume

 

故障处理5【 Unable to create patchObject】

 
	 ---------------------------Patching Failed---------------------------------
	Command execution failed during patching in home: /home/db/oracle/product/12.2.0, host: bjdb19c1.
	Command failed:  /home/db/oracle/product/12.2.0/OPatch/opatchauto  apply /home/backupfile/orapatch/30501910 -oh /home/db/oracle/product/12.2.0 -target_type rac_database -binary -invPtrLoc /home/db/oracle/product/12.2.0/oraInst.loc -jre /home/db/oracle/product/12.2.0/OPatch/jre -persistresult /home/db/oracle/product/12.2.0/OPatch/auto/dbsessioninfo/sessionresult_bjdb19c1_rac.ser -analyzedresult /home/db/oracle/product/12.2.0/OPatch/auto/dbsessioninfo/sessionresult_analyze_bjdb19c1_rac.ser
	Command failure output: 
	==Following patches FAILED in apply:

	Patch: /home/backupfile/orapatch/30501910/30489227
	Log: 
	Reason: Failed during Patching: oracle.opatch.opatchsdk.OPatchException: Unable to create patchObject
	Possible causes are:
	   ORACLE_HOME/inventory/oneoffs/30489227 is corrupted. java.lang.RuntimeException: No Patch exists,Please check. 

	After fixing the cause of failure Run opatchauto resume

	]
	OPATCHAUTO-68061: The orchestration engine failed.
	OPATCHAUTO-68061: The orchestration engine failed with return code 1
	OPATCHAUTO-68061: Check the log for more details.
	OPatchAuto failed.
	 
 
	 
	-------
	 
	 最后找到了问题的根源，是2节点$ORACLE_HOME/inventory/oneoffs/下的文件有问题。也就是说是Local Inventory出了问题。

	将1节点正确的$ORACLE_HOME/inventory/oneoffs/下的文件复制到2节点：
	su - oracle
	$ cd /home/db/oracle/product/12.2.0/inventory/oneoffs
	$ scp * bjdb19c0:/home/db/oracle/product/12.2.0/inventory/oneoffs

	 

	2节点验证补丁集：

	[oracle@node2 OPatch]$ ./opatch lspatches
	29850993;OCW RELEASE UPDATE 19.4.0.0.0 (29850993)
	29834717;Database Release Update : 19.4.0.0.190716 (29834717)

	OPatch succeeded.

	 

	这样复制会可能会导致缺少某个组件使得oracle出现问题，需要再次打2节点db的补丁。


		

	[root@node2 ~]# /u01/app/oracle/product/19.3.0/dbhome_1/OPatch/opatchauto apply /soft/29708769 -oh /u01/app/oracle/product/19.3.0/dbhome_1
		......
	---------------------------Patching Failed---------------------------------
	Command execution failed during patching in home: /u01/app/oracle/product/19.3.0/dbhome_1, host: node2.
	Command failed:  /u01/app/oracle/product/19.3.0/dbhome_1/OPatch/opatchauto  apply /soft/29708769 -oh /u01/app/oracle/product/19.3.0/dbhome_1 -target_type rac_database -binary -invPtrLoc /u01/app/oracle/product/19.3.0/dbhome_1/oraInst.loc -jre /u01/app/oracle/product/19.3.0/dbhome_1/OPatch/jre -persistresult /u01/app/oracle/product/19.3.0/dbhome_1/OPatch/auto/dbsessioninfo/sessionresult_node2_rac.ser -analyzedresult /u01/app/oracle/product/19.3.0/dbhome_1/OPatch/auto/dbsessioninfo/sessionresult_analyze_node2_rac.ser
	Command failure output: 
	==Following patches FAILED in apply:

	Patch: /soft/29708769/29850993
	Log: /u01/app/oracle/product/19.3.0/dbhome_1/cfgtoollogs/opatchauto/core/opatch/opatch2019-09-03_01-54-29AM_1.log
	Reason: Failed during Patching: oracle.opatch.opatchsdk.OPatchException: ApplySession failed in system modification phase... ‘’'ApplySession::apply failed: java.io.IOException: 
	oracle.sysman.oui.patch.PatchException: java.io.FileNotFoundException: /u01/app/oraInventory/ContentsXML/oui-patch.xml (Permission denied)' 

	After fixing the cause of failure Run opatchauto resume

	]
	OPATCHAUTO-68061: The orchestration engine failed.
	OPATCHAUTO-68061: The orchestration engine failed with return code 1
	OPATCHAUTO-68061: Check the log for more details.
	OPatchAuto failed.

	OPatchauto session completed at Tue Sep  3 01:55:33 2019
	Time taken to complete the session 4 minutes, 4 seconds

	 opatchauto failed with error code 42

	   

	这里报错/u01/app/oraInventory/ContentsXML/oui-patch.xml (Permission denied)，对比一节点和二节点文件的权限找到问题。

	[root@node1 ~]# ll/u01/app/oraInventory/ContentsXML/oui-patch.xml
	-rw-rw---- 1 grid   oinstall 174 Sep  2 20:11 oui-patch.xml

	[root@node2 ~]# ll /u01/app/oraInventory/ContentsXML/oui-patch.xml
	-rw-r--r-- 1 grid   oinstall 174 Sep  2 20:55 oui-patch.xml
	 

	两个节点该文件权限不一致导致。

	[root@node2 ContentsXML]# chmod 660 oui-patch.xml

	 

	将权限改好之后再次打2节点db补丁。

	[root@node2 OPatch]# /u01/app/oracle/product/19.3.0/dbhome_1/OPatch/opatchauto apply /soft/29708769 -oh /u01/app/oracle/product/19.3.0/dbhome_1

	OPatchauto session is initiated at Tue Sep  3 19:39:45 2019

	......
	 
	OPatchAuto successful.

	--------------------------------Summary--------------------------------

	Patching is completed successfully. Please find the summary as follows:

	Host:node2
	RAC Home:/u01/app/oracle/product/19.3.0/dbhome_1
	Version:19.0.0.0.0
	Summary:

	==Following patches were SKIPPED:

	Patch: /soft/29708769/29851014
	Reason: This patch is not applicable to this specified target type - "rac_database"

	Patch: /soft/29708769/29401763
	Reason: This patch is not applicable to this specified target type - "rac_database"

	Patch: /soft/29708769/29850993
	Reason: This patch is already been applied, so not going to apply again.

	Patch: /soft/29708769/29834717
	Reason: This patch is already been applied, so not going to apply again.

	OPatchauto session completed at Tue Sep  3 19:41:30 2019
	Time taken to complete the session 1 minute, 46 seconds

	 

	到这里补丁已经成功打上了！

	[oracle@node2 OPatch]$ ./opatch lspatches
	29850993;OCW RELEASE UPDATE 19.4.0.0.0 (29850993)
	29834717;Database Release Update : 19.4.0.0.190716 (29834717)

	OPatch succeeded.

故障处理6【requires prerequisite patch(es) [29834717]】

	UtilSession failed: Interim patch 30396974  requires prerequisite patch(es) [29834717] which are not present in the Oracle Home.
	Apply prerequisite patch(es) [29834717] before applying interim patch 30396974.


	-----下载
	-rw-r--r-- 1 root root     1986870968 Aug  7 12:04 p29699097_190000_Linux-x86-64.zip

	# cd /home/backupfile/orapath
	# unzip p29699097_190000_Linux-x86-64.zip
	# chmod  777 *


	# /home/db/oracle/product/12.2.0/OPatch/opatchauto apply /home/backupfile/orapatch/29699097/29708769 -oh /home/db/oracle/product/12.2.0
		--------------------------------Summary--------------------------------

		Patching is completed successfully. Please find the summary as follows:

		Host:bjdb19c1
		RAC Home:/home/db/oracle/product/12.2.0
		Version:19.0.0.0.0
		Summary:

		==Following patches were SKIPPED:

		Patch: /home/backupfile/orapatch/29699097/29708769/29851014
		Reason: This patch is not applicable to this specified target type - "rac_database"

		Patch: /home/backupfile/orapatch/29699097/29708769/29401763
		Reason: This patch is not applicable to this specified target type - "rac_database"

		Patch: /home/backupfile/orapatch/29699097/29708769/29850993
		Log: /home/db/oracle/product/12.2.0/cfgtoollogs/opatchauto/core/opatch/opatch2020-08-07_12-41-25PM_1.log
		Reason: /home/backupfile/orapatch/29699097/29708769/29850993 is not required to be applied to oracle home /home/db/oracle/product/12.2.0

		Patch: /home/backupfile/orapatch/29699097/29708769/29834717
		Log: /home/db/oracle/product/12.2.0/cfgtoollogs/opatchauto/core/opatch/opatch2020-08-07_12-41-25PM_1.log
		Reason: /home/backupfile/orapatch/29699097/29708769/29834717 is not required to be applied to oracle home /home/db/oracle/product/12.2.0


	su - oracle
	$ opatch lspatches
	Patch: /home/backupfile/orapatch/29699097/29708769/29834717

	30557433;Database Release Update : 19.6.0.0.200114 (30557433)
	30489227;OCW RELEASE UPDATE 19.6.0.0.0 (30489227)