$$\begin{align} \int\sec^3(x)\tan^2(x)\,dx&=\int\sec^3(x)(\sec^2(x)-1)\,dx \\ &=\int\sec^5(x)\,dx-\int\sec^3(x)\,dx \end{align}$$
$$\int\sec^3(x)\,dx=\frac12(\sec(x)\tan(x)+\ln|\sec(x)+\tan(x)|)+C$$
$$\int\sec^5(x)\,dx$$ Let $u=\sec^3(x)$, then $du=3\sec^3(x)\tan(x)\,dx $. Let $dv=\sec^2(x)\,dx$, then $v=\tan(x)$
$$\begin{align} \int\sec^5(x)\,dx&=\sec^3(x)\tan(x)-3\int\sec^3(x)(\sec^2(x)-1)\,dx \\ &=\sec^3(x)\tan(x)-3\int\sec^5(x)\,dx+3\int\sec^3(x)\,dx \\ &=\frac14\sec^3(x)\tan(x)+\frac38(\sec(x)\tan(x)+\ln|\sec(x)+\tan(x)|)+C \end{align}$$
$$\int\sec^5(x)\,dx-\int\sec^3(x)\,dx=\frac14\sec^3(x)\tan(x)-\frac18(\sec(x)\tan(x)+\ln|\sec(x)+\tan(x)|)+C$$