Trying to show using a different approach that $\int_0^1 \frac{\sqrt x \ln x}{x^2-x+1}dx =\frac{\pi^2\sqrt 3}{9}-\frac{8}{3}G\, $ I have stumbled upon this series: $$\sum_{n=1}^\infty \frac{\sin\left(n\frac{\pi}{3}\right)}{(2n+1)^2}$$ The linked proof relies upon this trigamma identity. Now by rewriting the integral as: $$I=\int_0^1 \frac{\sqrt{x}\ln x}{x^2-2\cos\left(\frac{\pi}{3}\right)x+1}dx$$ And using that: $$\frac{\sin t}{x^2-2x\cos t+1}=\frac{1}{2i}\left(\frac{e^{it}}{1-xe^{it}}-\frac{e^{-it}}{1-xe^{-it}}\right)=\Im \left(\frac{e^{it}}{1-xe^{it}}\right)=$$ $$=\sum_{n=0}^{\infty} \Im\left(x^n e^{i(n+1)t}\right)=\sum_{n=0}^\infty x^n\sin((n+1)t)$$ $$I=\frac{1}{\sin \left(\frac{\pi}{3}\right)}\sum_{n=0}^\infty \sin\left(\frac{\pi}{3} (n+1) \right)\int_0^1 x^{n+1/2} \ln x dx$$ $$\text{Since} \ \int_0^1 x^p \ln x dx= -\frac{1}{(p+1)^2}$$ $$I=-\frac{2}{\sqrt 3} \sum_{n=0}^\infty \frac{\sin\left((n+1)\frac{\pi}{3}\right)}{(n+1+1/2)^2}=-\frac{8}{\sqrt 3}\sum_{n=1}^\infty \frac{\sin\left(n\frac{\pi}{3}\right)}{(2n+1)^2} $$ And well by using the previous link we can deduce that the series equals to $\frac{G}{\sqrt 3} -\frac{\pi^2}{24}$, where $G$ is Catalan's constant.
I thought this might be a coefficient of some Fourier series, or taking the imaginary part of $\left(\sum_{n=1}^\infty \frac{e^{i\frac{n\pi}{3}}}{(2n+1)^2}\right)$, but I was not that lucky afterwards.
Is there a way to show the result without relying on that trigamma identity? Another approach to the integral would of course be enough.
\begin{align*} I&=\int_0^1 \frac{\sqrt{x}\ln x} {x^2-x+1}\,dx \end{align*}
Perform the change of variable $y=\sqrt{x}$,
\begin{align*} J&=4\int_0^1 \frac{x^2\ln x} {x^4-x^2+1}\,dx\\ &=\left[\left(\frac{1}{\sqrt{3}}\ln\left(\frac{x^2-\sqrt{3}x+1}{x^2+\sqrt{3}x+1}\right)+2\arctan\left(\frac{x}{1-x^2}\right)\right)\ln x\right]_0^1-\\ &\int_0^1 \frac{1}{x\sqrt{3}}\ln\left(\frac{x^2-\sqrt{3}x+1}{x^2+\sqrt{3}x+1}\right)\,dx-2\int_0^1\frac{1}{x}\arctan\left(\frac{x}{1-x^2}\right)\,dx\\ &=-\int_0^1 \frac{1}{x\sqrt{3}}\ln\left(\frac{x^2-\sqrt{3}x+1}{x^2+\sqrt{3}x+1}\right)\,dx-2\int_0^1\frac{1}{x}\arctan\left(\frac{x}{1-x^2}\right)\,dx\\ \end{align*}
\begin{align*} L&=\int_0^1 \frac{1}{x}\ln\left(\frac{x^2-\sqrt{3}x+1}{x^2+\sqrt{3}x+1}\right)\,dx\\ &=\int_0^1 \frac{1}{x}\ln\left(\frac{\left(x^2-\sqrt{3}x+1\right)\left(x^2+\sqrt{3}x+1\right)}{\left(x^2+\sqrt{3}x+1\right)^2}\right)\,dx\\ &=\int_0^1 \frac{1}{x}\ln\left(x^4-x^2+1\right)\,dx-2\int_0^1 \frac{1}{x}\ln\left(x^2+\sqrt{3}x+1)\right)\,dx\\ &=\int_0^1 \frac{x}{x^2}\ln\left(\frac{1+x^6}{1+x^2}\right)\,dx-2\int_0^1 \frac{1}{x}\ln\left(x^2+\sqrt{3}x+1)\right)\,dx\\ \end{align*}
In the first integral perform the change of variable $y=x^2$,
\begin{align*}L&=\frac{1}{2}\int_0^1 \frac{1}{x}\ln\left(\frac{1+x^3}{1+x}\right)\,dx-2\int_0^1 \frac{1}{x}\ln\left(x^2+\sqrt{3}x+1\right)\,dx\\ &=\frac{1}{2}\int_0^1 \frac{x^2}{x^3}\ln\left(1+x^3\right)\,dx-\frac{1}{2}\int_0^1 \frac{1}{x}\ln\left(1+x\right)\,dx-2\int_0^1 \frac{1}{x}\ln\left(x^2+\sqrt{3}x+1\right)\,dx\\ \end{align*}
In the first integral perform the change of variable $y=x^3$,
\begin{align*}L&=\frac{1}{6}\int_0^1 \frac{1}{x}\ln\left(1+x\right)\,dx-\frac{1}{2}\int_0^1 \frac{1}{x}\ln\left(1+x\right)\,dx-2\int_0^1 \frac{1}{x}\ln\left(x^2+\sqrt{3}x+1\right)\,dx\\ &=-\frac{1}{3}\int_0^1 \frac{1}{x}\ln\left(1+x\right)\,dx-2\int_0^1 \frac{1}{x}\ln\left(x^2+\sqrt{3}x+1\right)\,dx\\ &=\frac{1}{3}\int_0^1 \frac{1}{x}\ln\left(1-x\right)\,dx-\frac{1}{3}\int_0^1 \frac{1}{x}\ln\left(1-x^2\right)\,dx-2\int_0^1 \frac{1}{x}\ln\left(x^2+\sqrt{3}x+1\right)\,dx\\ \end{align*}
In the second integral perform the change of variable $y=x^2$,
\begin{align*}L&=\frac{1}{3}\int_0^1 \frac{1}{x}\ln\left(1-x\right)\,dx-\frac{1}{6}\int_0^1 \frac{1}{x}\ln\left(1-x\right)\,dx-2\int_0^1 \frac{1}{x}\ln\left(x^2+\sqrt{3}x+1\right)\,dx\\ &=\frac{1}{6}\int_0^1 \frac{1}{x}\ln\left(1-x\right)\,dx-2\int_0^1 \frac{1}{x}\ln\left(x^2+\sqrt{3}x+1\right)\,dx\\ \end{align*}
\begin{align*}M&=\int_0^1 \frac{1}{x}\ln\left(x^2+\sqrt{3}x+1\right)\,dx\\\end{align*}
Define the function $F$ on $\left[0;\sqrt{3}\right]$ by,
\begin{align*}F(a)=\int_0^1 \frac{1}{x}\ln\left(x^2+ax+1\right)\,dx\\\end{align*}
Observe that $F(\sqrt{3})=M$ and,
\begin{align*}F(0)&=\int_0^1 \frac{1}{x}\ln\left(x^2+1\right)\,dx\\ &=-\frac{1}{4}\int_0^1 \frac{1}{x}\ln\left(1-x\right)\,dx\\ \end{align*}
Since $0<a<2$,
\begin{align*}F^\prime(a)&=\int_0^1 \frac{1}{x^2+ax+1}\,dx\\ &=\left[\frac{2}{\sqrt{4-a^2}}\arctan\left(\frac{2x+a}{\sqrt{4-a^2}}\right)\right]_0^1\\ &=\frac{2}{\sqrt{4-a^2}}\arctan\left(\frac{2+a}{\sqrt{4-a^2}}\right)-\frac{2}{\sqrt{4-a^2}}\arctan\left(\frac{a}{\sqrt{4-a^2}}\right)\\ &=\frac{2}{\sqrt{4-a^2}}\arctan\left(\sqrt{\frac{2-a}{2+a}}\right)\\ \end{align*}
\begin{align*}F(\sqrt{3})-F(0)=\int_0^{\sqrt{3}}\frac{2}{\sqrt{4-a^2}}\arctan\left(\sqrt{\frac{2-a}{2+a}}\right)\,da\\ \end{align*}
Perform the change of variable $y=\sqrt{\frac{2-a}{2+a}}$,
\begin{align*}F(\sqrt{3})-F(0)&=4\int_{2-\sqrt{3}}^1 \frac{\arctan y}{1+y^2}\,dy\\ &=2\Big[\arctan^2(x)\Big]_{2-\sqrt{3}}^1\\ &=2\times \frac{\pi^2}{4^2} -2\times \frac{\pi^2}{12^2}\\ &=\frac{\pi^2}{9} \end{align*}
Since,
\begin{align*}\int_0^1 \frac{1}{x}\ln\left(1-x\right)\,dx=-\frac{\pi^2}{6}\end{align*}
Then, \begin{align*}M&=F(\sqrt{3})\\ &=F(0)+\frac{\pi^2}{9}\\ &=-\frac{1}{4}\times -\frac{\pi^2}{6}+\frac{\pi^2}{9}\\ &=\frac{11\pi^2}{72} \end{align*}
Therefore,
\begin{align*}L&=-\frac{1}{6}\times -\frac{\pi^2}{6}-2M\\ &=\frac{1}{6}\times -\frac{\pi^2}{6}-2\times \frac{11\pi^2}{72}\\ &=-\frac{\pi^2}{3} \end{align*}
\begin{align*} K&=\int_0^1 \frac{\arctan\left(\frac{x}{1-x^2}\right)}{x}\,dx\\ \end{align*}
Perform the change of variable $x=\tan\left(\frac{t}{2}\right) $,
\begin{align*} K&=\int_0^{\frac{\pi}{2}} \frac{\arctan\left(\frac{1}{2}\tan t\right)}{\sin t}\,dt \end{align*}
Define the function $H$ on $\left[\frac{1}{2};1\right]$ by,
\begin{align*}H(a)&=\int_0^{\frac{\pi}{2}} \frac{\arctan\left(a\tan t\right)}{\sin t}\,dt\end{align*}
Observe that $K=H\left(\dfrac{1}{2}\right)$ and,
\begin{align*}H(1)&=\int_0^{\frac{\pi}{2}} \frac{t}{\sin t}\,dt\\ &=\Big[t\ln\left(\tan\left(\frac{t}{2} \right)\right)\Big]_0^{\frac{\pi}{2}}-\int_0^{\frac{\pi}{2}}\ln\left(\tan\left(\frac{t}{2} \right)\right)\,dt\\ &=-\int_0^{\frac{\pi}{2}}\ln\left(\tan\left(\frac{t}{2} \right)\right)\,dt\\ \\\end{align*}
Perform the change of variable $x=\dfrac{t}{2}$,
\begin{align*}H(1)&=-2\int_0^{\frac{\pi}{4}}\ln\left(\tan\left(t \right)\right)\,dt\\ &=2\text{G} \\\end{align*}
\begin{align*}H^\prime (a)&=\int_0^{\frac{\pi}{2}} \frac{\cos x}{1-(1-a^2)\sin^2 x}\,dt\\ &=\left[\frac{1}{2\sqrt{1-a^2}}\ln\left(\frac{1+\sin(x)\sqrt{1-a^2}}{1-\sin(x)\sqrt{1-a^2}}\right)\right]_0^{\frac{\pi}{2}}\\ &=\frac{1}{2\sqrt{1-a^2}}\ln\left(\frac{1+\sqrt{1-a^2}}{1-\sqrt{1-a^2}}\right) \end{align*}
Therefore,
\begin{align*}H(1)-H\left(\frac{1}{2}\right)&=\int_{\frac{1}{2}}^1 \frac{1}{2\sqrt{1-a^2}}\ln\left(\frac{1+\sqrt{1-a^2}}{1-\sqrt{1-a^2}}\right)\,da\end{align*}
Perform the change of variable $y=\arctan\left(\sqrt{\dfrac{1+\sqrt{1-x^2}}{1-\sqrt{1-x^2}}}\right)$
\begin{align*}H(1)-H\left(\frac{1}{2}\right)&=-2\int_{\frac{\pi}{12}}^{\frac{\pi}{4}} \ln\left(\tan y\right)\,dy\\ &=-2\int_0^{\frac{\pi}{4}} \ln\left(\tan y\right)\,dy+\int_0^{\frac{\pi}{12}} \ln\left(\tan y\right)\,dy \end{align*}
But, it's well known that:
\begin{align*}\int_0^{\frac{\pi}{4}} \ln\left(\tan y\right)\,dy=-\text{G}\\ \int_0^{\frac{\pi}{12}} \ln\left(\tan y\right)\,dy=-\frac{2}{3}\text{G}\\ \end{align*}
(see Integral: $\int_0^{\pi/12} \ln(\tan x)\,dx$ )
Therefore,
\begin{align*}H(1)-H\left(\frac{1}{2}\right)&=2\text{G}+2\times -\frac{2}{3}\text{G}\\ &=\frac{2}{3}\text{G} \end{align*}
Thus,
\begin{align*} K&=H\left(\frac{1}{2}\right)\\ &=H(1)-\frac{2}{3}\text{G}\\ &=2\text{G}-\frac{2}{3}\text{G}\\ &=\frac{4}{3}\text{G} \end{align*}
\begin{align*}I&=-\frac{1}{\sqrt{3}}L-2K\\ &=-\frac{1}{\sqrt{3}}\times -\frac{\pi^2}{3}-2\times \frac{4}{3}\text{G}\\ &=\boxed{\frac{\sqrt{3}\pi^2}{9}-\frac{8}{3}\text{G}} \end{align*}
PS:
Actually,
\begin{align}\int_0^1 \frac{\arctan\left( \frac{x}{1-x^2}\right)}{x}\,dx-\int_0^1 \frac{\arctan x}{x}\,dx=\int_0^1 \frac{\arctan \left(x^3\right)}{x}\,dx\end{align}
In the latter integral perform the change of variable $y=x^3$,
\begin{align}\int_0^1 \frac{\arctan\left( \frac{x}{1-x^2}\right)}{x}\,dx-\int_0^1 \frac{\arctan x}{x}\,dx=\frac{1}{3}\int_0^1 \frac{\arctan x}{x}\,dx\end{align}
therefore,
\begin{align}\int_0^1 \frac{\arctan\left( \frac{x}{1-x^2}\right)}{x}\,dx&=\frac{1}{3}\int_0^1 \frac{\arctan x}{x}\,dx+\int_0^1 \frac{\arctan x}{x}\,dx\\ &=\frac{4}{3}\int_0^1 \frac{\arctan x}{x}\,dx\\ &=\frac{4}{3}\text{G} \end{align}