Ha ha I found the solution.

I looked more closely at mouse_trace() and found it returns a distance, not a vector so instead I used the to[3] vector to rotate the cannon and it works.

Code:

---

action tankTurret
{
	turret = my;
	while(!player){ wait(1);}	//wait for the player entity to load before doing anything
	while(player)					//once the player exists
	{
		my.x = player.x;			//follow player
		my.y = player.y;
		my.z = player.z;
		mouse_trace();
		vec_diff(to,to,my.x);
		vec_to_angle(my.pan,to);
		if((my.tilt != 0) || (my.roll != 0)) { my.tilt = 0; my.roll = 0;} //stay aligned with horizontal plane
		wait(1);
	}
}

  

---

no changes were made to mouse_trace();