euler = orientation;

What is orientation ?

JoeJ said:
euler = orientation;
world = glm::eulerAngleZ(orientation[2]);
world *= glm::eulerAngleY(orientation[1]);
world *= glm::eulerAngleX(orientation[0]);

I don't understand that part.

‘orientation’ is a vec3 of euler angles.

Like your code, it calculates X,Y,Z rotation matrices from them and multiplicates them together in given order.

All right. I don't think I understand what Euler's angles mean to me.

Euler angles means a series of 3 rotations. Take your current code as example: You make 3 rotation matrices and you use them all to calculate a final rotation of objects.
(Notice we could make this final rotation in one step, for example using an axis and angle of rotation.)

So you already use euler angles, but the problem is that the resulting rotation is applied to whatever the orientation of the object is, and this results in unwanted camera tilting.

The easiest way to avoid this is to keep track of the 3 angles, and to create camera orientation from scratch form those angles each time. This way you have precise control over camera behavior and you can prevent tilting in the first place.

Using your code base it should look like this:

void UpdateCamera(vec3 cameraOrnMatrix[3], vec3 cameraOrnMatrixInversed[3])
{
static float cameraEulerX = 0;
static float cameraEulerY = 0;
static float cameraEulerZ = 0;
if (ButtonPressed('A')) cameraEulerY += 0.01f;
if (ButtonPressed('D')) cameraEulerY -= 0.01f;
if (ButtonPressed('W')) cameraEulerX += 0.01f;
if (ButtonPressed('S')) cameraEulerX -= 0.01f;
// we leave Z at zero to prevent tilting

vec3 rotX[3]; MakeRotationX(rotX, cameraEulerX);
vec3 rotY[3]; MakeRotationY(rotY, cameraEulerY);
vec3 rotZ[3]; MakeRotationZ(rotZ, cameraEulerZ);

CopyMatrix (cameraOrnMatrix, rotZ);
MultiplyMatrix (cameraOrnMatrix, rotY);
MultiplyMatrix (cameraOrnMatrix, rotX);

TransposeMatrix(cameraOrnMatrixInversed, cameraOrnMatrix); // personally i use the inverse to setup camera projection, but depends on how you do it.
}

This is my function :

void transpose_mat_3x3(t_vec3 *dst, t_vec3 src[3])
{
	dst[0].x = src[0].x;
	dst[0].y = src[1].x;
	dst[0].z = src[2].x;
	dst[1].x = src[0].y;
	dst[1].y = src[1].y;
	dst[1].z = src[2].y;
	dst[2].x = src[0].z;
	dst[2].y = src[1].z;
	dst[2].z = src[2].z;
}

void	MultiplyMatrix(t_vec3 *lhs, t_vec3 *rhs)
{
	t_vec3 *tmp;

	tmp = lhs;
	lhs[0] = rotate_vector(tmp, rhs[0]);
	lhs[1] = rotate_vector(tmp, rhs[1]);
	lhs[2] = rotate_vector(tmp, rhs[2]);
}

void update_camera(t_vec3 cameraOrnMatrix[3], int key)
{
	static float cameraEulerX = 0;
	static float cameraEulerY = 0;
	static float cameraEulerZ = 0;
	t_vec3 rotX[3];
	t_vec3 rotY[3];
	t_vec3 rotZ[3];

	if (key == KEY_H)
		cameraEulerY += 0.08f;
	else if (key == KEY_F)
		cameraEulerY -= 0.08f;
	else if (key == KEY_G)
		cameraEulerX += 0.08f;
	else if (key == KEY_T)
		cameraEulerX -= 0.08f;
	// we leave Z at zero to prevent tilting

	make_rotation_x(rotX, cameraEulerX);
	make_rotation_y(rotY, cameraEulerY);
	make_rotation_z(rotZ, cameraEulerZ);

	cpy_matrix(cameraOrnMatrix, rotZ);
	MultiplyMatrix(cameraOrnMatrix, rotY);
	MultiplyMatrix(cameraOrnMatrix, rotX);

	//transpose_mat_3x3(cameraOrnMatrixInversed, cameraOrnMatrix); // personally i use the inverse to setup camera projection, but depends on how you do it.

	cameraOrnMatrix[2] = invert(cameraOrnMatrix[2]);
}

void		move_cam(t_specs *s, int key)
{
	t_camera	*cam;
	t_vec3		orientation[3];

	cam = s->current_cam;
	orientation[0] = cam->right;
	orientation[1] = cam->up;
	orientation[2] = cam->vec;
	if (key == KEY_H || key == KEY_F || key == KEY_G || key == KEY_T || key == KEY_R || key == KEY_Y)
	{
		update_camera(orientation, key);
		//t_vec3 *tmp = orientation;
		//transpose_mat_3x3(orientation, tmp);
	}
	s->current_cam->right = orientation[0];
	s->current_cam->up = orientation[1];
	s->current_cam->vec = orientation[2];
	normalize(&s->current_cam->right);
	normalize(&s->current_cam->up);
	normalize(&s->current_cam->vec);
}

But I have to add this line in update_camera function to fix the effect :

cameraOrnMatrix[2] = invert(cameraOrnMatrix[2]);

I always rotate to the left. I didn't speed the video to fully understand.
Before adding the line :

And after :

Maybe it comes from my function get_ray_dir() which gives the direction of the ray from the camera space to the world space.

static t_vec3	orient_camera(t_camera *cam, t_vec3 ray_coord)
{
	t_vec3	forward;
	t_vec3	ray_dir;

	forward = get_normalised(invert(cam->vec));
	ray_dir.x = ray_coord.x * cam->right.x + ray_coord.y * cam->up.x
			+ ray_coord.z * forward.x;
	ray_dir.y = ray_coord.x * cam->right.y + ray_coord.y * cam->up.y
			+ ray_coord.z * forward.y;
	ray_dir.z = ray_coord.x * cam->right.z + ray_coord.y * cam->up.z
			+ ray_coord.z * forward.z;
	return (ray_dir);
}

static t_vec3	get_ray_dir(int i, int j, t_resolution res, t_camera *cam)
{
	t_ray ray;
	float scale;
	float point_x;
	float point_y;
	float ratio;

	ratio = res.x / (float)res.y;
	scale = tan(cam->fov / 2 * M_PI / 180);
	point_x = (2 * (j + 0.5) / (float)res.x - 1) * ratio * scale;
	point_y = (1 - 2 * (i + 0.5) / res.y) * scale;
	ray.coord = (t_vec3){point_x, point_y, -1};
	ray.dir = orient_camera(cam, ray.coord);
	normalize(&ray.dir);
	return (ray.dir);
}

This is how the camera axis are initialized at the beginning of the program (do it once) :

void	init_camera_vecs(t_camera *cam)
{
	/*t_vec3	forward;

	forward = get_normalised(invert(cam->vec));
	cam->up = (t_vec3){0, 0, 0};
	if (fabs(cam->vec.y) > 0.7)
		cam->right = vec_cross((t_vec3){0, 0, -1}, forward);
	else
		cam->right = vec_cross((t_vec3){0, 1, 0}, forward);
	cam->up = vec_cross(forward, cam->right);*/

	//////////////////////////
	 cam->up =  (t_vec3){0, 1, 0};
	 cam->right =  (t_vec3){1, 0, 0};
	/////////////////////////
}

This way, the current camera orientation is not taken into account ?

ellenature said:
But I have to add this line in update_camera function to fix the effect : cameraOrnMatrix[2] = invert(cameraOrnMatrix[2]);

What is invert doing? Reversing Z axis i guess, so changing left / right handed coordinate system.

Maybe the reason is this:

////////////////////////// 
cam->up = (t_vec3){0, 1, 0}; 
cam->right = (t_vec3){1, 0, 0}; 
///////////////////////// 

probably you want to set the front vector here too, and depending on if it's (0,0,1) or (0,0,-1) you define left / right handed for the camera. (Another trial and error case.)

But not sure if this can explain the stretching seen in the first video.

You could write a tool function to verfiy all your matrices have the same left / right handed convention:

vec temp = Cross(matrix[0], matrix[1])
return Dot(temp, matrix[2]) > 0;

It should give the same result for all matrices, as long as you do not use negative scaling to mirror things.

I succeeded by following this:
https://gamedev.stackexchange.com/questions/185245/rotating-camera-in-3d-without-yawing-diagonally

Moreover, I have stored the angles in my structure rather than defining them statically because I have several cameras.

float			wrap_angle(float angle)
{
	float two_pi;

	two_pi = 2 * M_PI;
	return (angle - two_pi * floor((angle + M_PI) / two_pi));
}

void	init_camera_angles(t_camera *cam)
{
	cam->yaw = atan2(cam->vec.x, -cam->vec.z);
	cam->pitch = asin(cam->vec.y / sqrt(get_norm_2(cam->vec)));
	cam->roll = 0;
}

void        move_cam(t_specs *s, int key)
{
t_camera    *cam;
t_vec3      orientation[3];

cam = s->current_cam;
orientation[0] = cam->right;
orientation[1] = cam->up;
orientation[2] = cam->forward;

if (key == KEY_W)
    cam->coord = vec_add(cam->coord, cam->up);
else if (key == KEY_S)
    cam->coord = vec_sub(cam->coord, cam->up);
else if (key == KEY_A)
    cam->coord = vec_sub(cam->coord, cam->right);
else if (key == KEY_D)
    cam->coord = vec_add(cam->coord, cam->right);
else if (key == KEY_Q)
    cam->coord = vec_sub(cam->coord, cam->vec);
else if (key == KEY_E)
    cam->coord = vec_add(cam->coord, cam->vec);
else if (key == KEY_H)
    cam->yaw += 0.1;
else if (key == KEY_F)
    cam->yaw -= 0.1;
else if (key == KEY_T)
    cam->pitch -= 0.1;
else if (key == KEY_G)
    cam->pitch += 0.1;
else if (key == KEY_R)
    cam->roll += 0.1;
else if (key == KEY_Y)
    cam->roll -= 0.1;
    
cam->yaw = wrap_angle(cam->yaw);
cam->pitch = wrap_angle(cam->pitch);
cam->roll = wrap_angle(cam->roll);

make_rotation_z(orientation, cam->roll);
make_rotation_x(rotation, cam->pitch);
mult_matrix(orientation, rotation, orientation);
make_rotation_y(rotation, cam->yaw);
mult_matrix(orientation, rotation, orientation);

s->current_cam->right = orientation[0];
s->current_cam->up = orientation[1];
s->current_cam->forward = orientation[2];
normalize(&s->current_cam->right);
normalize(&s->current_cam->up);
normalize(&s->current_cam->forward);
}

I have a problem at the beginning of the programme when I initialise the three vectors of my camera.
I use this function to initialise these three vectors:

void    init_camera_axis(t_camera *cam)
{
   t_vec3 up_tmp;

   cam->forward = get_normalised(invert(cam->vec));
   if (fabs(cam->vec.y) > 0.7)
   {
       if (cam->vec.z <= 0)
       up_tmp = (t_vec3){0, 0, 1};
       else
           up_tmp = (t_vec3){0, 0, -1};
   }
   else
       up_tmp = (t_vec3){0, 1, 0};
   cam->right = vec_cross(up_tmp, cam->forward);
   cam->up = vec_cross(cam->forward, cam->right);
}

The problem is that I get a right equal to (0.85, 0, 0) instead of (1, 0, 0), which produces the effect of a different zoom on the first image compared to the others generated with move_cam().

Maybe I'm doing the initialization of the three vectors wrong or I need to use another method?

Concerning the inversion, it was because I was using cam->vec instead of cam->forward. The inversion is now corrected.