There may be an easier way but that is what I can think of right now:
How could I calculate if H1 is inside triangle ACD
if(H1 is inside the square ACBD)
{
// calculate direction vector C->D
vec_diff(temp,C.x,D.x);
// calculate normal that points outside (in direction of B)
normal.x = temp.y;
normal.y = -temp.x;
normal.z = 0;
vec_normalize(normal,1);
// the normal may be pointing towards A, depending on where C and D are
if(vec_dot(A.x,normal) - vec_dot(C.x,normal) > 0) vec_inverse(normal); // or < 0, I'm not sure right now
// now check if H1 is on the same side of the direction vector temp
if(vec_dot(H1.x,normal) - vec_dot(C.x,normal) < 0) return 1;
}
return 0;