I was able to find
$$\int_0^{\frac{\pi}{2}}x^2 \cot x\ln(1-\sin x)\mathrm{d}x=-\frac14\sum_{n=1}^\infty\frac{4^n}{{2n\choose n}}\frac{H_{2n}}{n^3}$$ $$=5\operatorname{Li}_4\left(\frac12\right)-\frac{65}{32}\zeta(4)-2\ln^2(2)\zeta(2)+\frac5{24}\ln^4(2)$$
by converting it to the sum above then evaluating this sum but many integrals and sums were involved in the calculations.
Do you have a different idea to find this integral or its sum?
Here is my boring solution:
Differentiate both sides of \begin{equation*} \frac{1}{n{2n\choose n}}=\int_0^1\frac1x\left(\frac{x}{(1+x)^2}\right)^n\mathrm{d}x \end{equation*} with respect to $n$, \begin{gather*} \frac{d}{dn}\frac{1}{n{2n\choose n}}=\frac{d}{dn}\int_0^1\frac1x\left(\frac{x}{(1+x)^2}\right)^n\mathrm{d}x\\ =\int_0^1\frac1x\frac{\partial}{\partial n}\left(\frac{x}{(1+x)^2}\right)^n\mathrm{d}x\\ =\int_0^1\frac{1}x\ln\left(\frac{x}{(1+x)^2}\right)\left(\frac{x}{(1+x)^2}\right)^n\mathrm{d}x. \end{gather*} Let's find the derivative of $\frac{1}{n{2n\choose n}}$. By the definition of the binomial coefficient: \begin{equation*} {a\choose b}=\frac{\Gamma(a+1)}{\Gamma(b+1)\Gamma(a-b+1)}, \end{equation*} we have \begin{equation*} \frac{1}{n{2n\choose n}}=\frac1n\cdot\frac{\Gamma^2(n+1)}{\Gamma(2n+1)}. \end{equation*} Use $\Gamma(n+1)=n\Gamma(n)$, \begin{equation*} \frac{1}{n{2n\choose n}}=\frac1n\cdot\frac{n^2\Gamma^2(n)}{2n\Gamma(2n)}=\frac{\Gamma^2(n)}{2\Gamma(2n)}.\end{equation*} Differentiate both sides, \begin{gather*} \frac{d}{dn}\frac{1}{n{2n\choose n}}=\frac{d}{dn}\frac{\Gamma^2(n)}{2\Gamma(2n)}\\ \{\text{use $\Gamma'(n)=\Gamma(n)\psi(n)$ }\}\\ =\frac{2\Gamma(2n)\Gamma^2(n)\psi(n)-2\Gamma(2n)\Gamma^2(n)\psi(2n)}{2\Gamma^2(2n)}\\ =(\psi(n)-\psi(2n))\frac{\Gamma^2(n)}{\Gamma(2n)}\\ \{\text{use $\psi(n+1)=H_n-\gamma$}\}\\ =(H_{n-1}-\gamma-H_{2n-1}+\gamma)\frac{2}{n{2n\choose n}}\\ =\left(H_n-\frac1n-H_{2n}+\frac1{2n}\right)\frac{2}{n{2n\choose n}}\\ =\frac{2H_n}{n{2n\choose n}}-\frac{2H_{2n}}{n{2n\choose n}}-\frac{1}{n^2{2n\choose n}}. \end{gather*} Therefore, we have \begin{equation} \frac{2H_n}{n{2n\choose n}}-\frac{2H_{2n}}{n{2n\choose n}}-\frac{1}{n^2{2n\choose n}}=\int_0^1\frac{1}x\ln\left(\frac{x}{(1+x)^2}\right)\left(\frac{x}{(1+x)^2}\right)^n\mathrm{d}x.\label{shoot} \end{equation} Now multiply both sides by $\frac{4^n}{2n^2}$ then consider the summation, \begin{gather*} \sum_{n=1}^\infty\frac{4^n}{{2n\choose n}}\frac{H_{n}}{n^3}-\sum_{n=1}^\infty\frac{4^n}{{2n\choose n}}\frac{H_{2n}}{n^3}-\frac12\sum_{n=1}^\infty\frac{4^n}{{2n\choose n}}\frac{1}{n^4}\\ =\frac12\int_0^1\frac{1}x\ln\left(\frac{x}{(1+x)^2}\right)\left[\sum_{n=1}^\infty\frac{\left(\frac{4x}{(1+x)^2}\right)^n}{n^2}\right]\mathrm{d}x\\ =\frac12\int_0^1\frac{1}x\ln\left(\frac{x}{(1+x)^2}\right)\left[\operatorname{Li}_2\left(\frac{4x}{(1+x)^2}\right)\right]\mathrm{d}x\\ \overset{\text{IBP}}{=}-\frac54\zeta(4)-\frac12\int_0^1\left(\frac12\ln^2(x)+2\operatorname{Li}_2(-x)\right)\left[\frac{2(x-1)}{x(1+x)}\ln\left(\frac{1-x}{1+x}\right)\right]\mathrm{d}x\\ =-\frac54\zeta(4)+\frac12\underbrace{\int_0^1\frac{\ln^2(x)\ln(1-x)}{x}\mathrm{d}x}_{I_1}-\frac12\underbrace{\int_0^1\frac{\ln^2(x)\ln(1+x)}{x}\mathrm{d}x}_{I_2}\\ +2\underbrace{\int_0^1\frac{\ln(1-x)\operatorname{Li}_2(-x)}{x}\mathrm{d}x}_{I_3}-2\underbrace{\int_0^1\frac{\ln(1+x)\operatorname{Li}_2(-x)}{x}\mathrm{d}x}_{I_4}\\ -\underbrace{\int_0^1\frac{\ln^2(x)\ln(1-x)}{1+x}\mathrm{d}x}_{I_5}+\underbrace{\int_0^1\frac{\ln^2(x)\ln(1+x)}{1+x}\mathrm{d}x}_{I_6}\\ -4\underbrace{\int_0^1\frac{\ln(1-x)\operatorname{Li}_2(-x)}{1+x}\mathrm{d}x}_{I_7}+4\underbrace{\int_0^1\frac{\ln(1+x)\operatorname{Li}_2(-x)}{1+x}\mathrm{d}x}_{I_8}. \end{gather*} For the first sum, \begin{equation} \sum_{n=1}^\infty\frac{4^n}{{2n\choose n}}\frac{H_{n}}{n^3}=-8\operatorname{Li}_4\left(\frac12\right)+\zeta(4)+8\ln^2(2)\zeta(2)-\frac{1}{3}\ln^4(2). \end{equation} For the third one, \begin{gather} \sum_{n=1}^\infty\frac{4^n}{{2n\choose n}}\frac{1}{n^4}=-4\sum_{n=1}^\infty\frac{4^n}{{2n\choose n}} \frac{1}{n^2}\int_0^1 x^{2n-1}\ln(x)\mathrm{d}x\\\ =-4\int_0^1\frac{\ln(x)}{x}\left(\sum_{n=1}^\infty\frac{4^n}{{2n\choose n}} \frac{x^{2n}}{n^2}\right)\mathrm{d}x\\\ =-8\int_0^1\frac{\ln(x)\arcsin^2 x}{x}\mathrm{d}x\\\ \overset{\text{IBP}}{=}8\int_0^1\frac{\ln^2(x)\arcsin x}{\sqrt{1-x^2}}\mathrm{d}x\\ \overset{x=\sin t}{=}8\int_0^{\frac{\pi}{2}}t\ln^2(\sin t)\mathrm{d}t\\ =8\operatorname{Li}_4\left(\frac{1}{2}\right)-\frac{19}{4}\zeta(4)+4\ln^2(2)\zeta(2)+\frac{1}{3}\ln^4(2). \end{gather} For $I_1$, expand $\ln(1-x)$ in series, \begin{gather} I_1=-\sum_{n=1}^\infty\frac1{n}\int_0^1 x^{n-1}\ln^2(x)\mathrm{d}x=-2\sum_{n=1}^\infty\frac{1}{n^4}=-2\zeta(4).\nonumber \end{gather} For $I_2$, expand $\ln(1+x)$ in series, \begin{gather} I_2=-\sum_{n=1}^\infty\frac{-1)^n}{n}\int_0^1 x^{n-1}\ln^2(x)\textrm{d}x=-2\sum_{n=1}^\infty\frac{(-1)^n}{n^4}=\frac74\zeta(4).\nonumber \end{gather} For $I_3$, expand $\operatorname{Li}_2(-x)$ in series, \begin{gather*} I_3=\sum_{n=1}^\infty\frac{(-1)^n}{n^2}\int_0^1 x^{n-1}\ln(1-x)\textrm{d}x=-\sum_{n=1}^\infty\frac{(-1)^nH_n}{n^3}\\ =-2\operatorname{Li_4}\left(\frac12\right)+\frac{11}4\zeta(4)-\frac74\ln(2)\zeta(3)+\frac12\ln^2(2)\zeta(2)-\frac{1}{12}\ln^4(2). \end{gather*} For $I_4$, \begin{equation*} I_4=-\frac12\operatorname{Li}_2^2(-1)=-\frac{5}{16}\zeta(4). \end{equation*}
For $I_5$, \begin{equation} \int_0^1\frac{\ln^2(x)\ln(1-x)}{1+x}\mathrm{d}x=-4\operatorname{Li}_4\left(\frac12\right)+\zeta(4)+\ln^2(2)\zeta(2)-\frac16\ln^4(2). \end{equation} For $I_6$, \begin{gather} \int_0^1\frac{\ln^2(x)\ln(1+x)}{1+x}\mathrm{d}x=4\operatorname{Li}_4\left(\frac12\right)-\frac{15}{4}\zeta(4)+\frac72\ln(2)\zeta(3)-\ln^2(2)\zeta(2)\nonumber\\ +\frac16\ln^4(2). \end{gather}
For $I_7$, expand $\frac{\operatorname{Li}_2(-x)}{1+x}$, \begin{gather*} I_7=-\sum_{n=1}^\infty (-1)^n H_{n-1}^{(2)}\int_0^1 x^{n-1}\ln(1-x)\mathrm{d}x\\ =\sum_{n=1}^\infty\frac{(-1)^nH_{n-1}^{(2)}H_n}{n}=\sum_{n=1}^\infty\frac{(-1)^n\left(H_n^{(2)}-\frac1{n^2}\right)H_n}{n}\\ =\sum_{n=1}^\infty\frac{(-1)^nH_n^{(2)}H_n}{n}-\sum_{n=1}^\infty\frac{(-1)^nH_n}{n^3}\\ =\frac{15}4\zeta(4)-\frac{21}{8}\ln(2)\zeta(3)+\frac34\ln^2(2)\zeta(2)-\frac{1}{6}\ln^4(2). \end{gather*} For $I_8$, apply integration by parts, \begin{gather*} I_8=-3\operatorname{Li}_4\left(\frac12\right)-\frac14\ln^2(2)\zeta(2)+\frac12\int_0^1\frac{\ln^3(1+x)}{x}\mathrm{d}x\\ =3\zeta(4)-\frac{21}{8}\ln(2)\zeta(3)+\frac12\ln^2(2)\zeta(2)-\frac18\ln^4(2). \end{gather*} Put all together,
\begin{equation} \sum_{n=1}^\infty\frac{4^n}{{2n\choose n}}\frac{H_{2n}}{n^3}=-20\operatorname{Li}_4\left(\frac12\right)+\frac{65}{8}\zeta(4)+8\ln^2(2)\zeta(2)-\frac56\ln^4(2). \end{equation}
The solution completes on writing
$$\int_0^{\frac{\pi}{2}}x^2 \cot x\ln(1-\sin x)\mathrm{d}x=\int_0^{1}\frac{\ln(1-x)}{x}\arcsin^2(x)\mathrm{d}x$$
$$=\int_0^{1}\frac{\ln(1-x)}{x}\left(\frac12\sum_{n=1}^\infty\frac{4^n}{{2n\choose n}} \frac{x^{2n}}{n^2}\right)\mathrm{d}x$$
$$=\frac12\sum_{n=1}^\infty\frac{4^n}{{2n\choose n}} \frac{1}{n^2}\int_0^1 x^{2n-1}\ln(1-x)\mathrm{d}x$$
$$=-\frac14\sum_{n=1}^\infty\frac{4^n}{{2n\choose n}}\frac{H_{2n}}{n^3}.$$