Thanks again for your input.

I have gotten the AABB-ray and triangle-ray intersection code working. Here I've selected a red piece using the mouse. I can select the red and black pieces, or the sphere, or the background. If there's multiple intersections, I always take the position that's closest to the main_camera.eye

It turns out that I was doing it all wrong.

Instead of transforming the triangle vertices on the CPU, and using an OBB intersection test, I am transforming the eye ray and doing an AABB intersection test. No need to transform the vertices on the CPU. The code is now one or two orders of magnitude faster.

The code to transform the eye ray is at https://stackoverflow.com/a/44648703/3634553

I'm stuck! I can only pick pieces where they don't occlude the sphere. Any glaring mistakes?

The whole code is at: https://github.com/sjhalayka/opengl4_stlview_shadow_map

The relevant code is:

ray = screen_coords_to_world_coords(x, y, win_x, win_y);

float t = 0;

bool first_assignment = true;

if (true == line_sphere_intersect(main_camera.eye, ray, vec3(0, 0, 0), 0.5f, t))
{
	collision_location = main_camera.eye + ray * t;

	col_loc = sphere;

	first_assignment = false;
}

for (size_t i = 0; i < player_game_piece_meshes.size(); i++)
{
	glm::mat4 inverse = glm::inverse(player_game_piece_meshes[i].model_mat);
	glm::vec4 start = inverse * glm::vec4(main_camera.eye, 1.0);
	glm::vec4 direction = inverse * glm::vec4(ray, 0.0);
	direction = glm::normalize(direction);

	if (true == player_game_piece_meshes[i].intersect_AABB(start, direction))
	{
		vec3 closest_intersection_point;

		if (true == player_game_piece_meshes[i].intersect_triangles(start, direction, closest_intersection_point))
		{
			if (first_assignment)
			{
				collision_location = closest_intersection_point;

				col_loc = player_game_piece;
				collision_location_index = i;

				first_assignment = false;
			}
			else
			{
				vec3 c0 = vec3(main_camera.eye) - closest_intersection_point;
				vec3 c1 = vec3(main_camera.eye) - collision_location;

				if (length(c0) < length(c1))
				{
					collision_location = closest_intersection_point;

					col_loc = player_game_piece;
					collision_location_index = i;
				}
			}
		}
	}
}

I was missing a few key steps:

ray = screen_coords_to_world_coords(x, y, win_x, win_y);

float t = 0;

bool first_assignment = true;
mat4 collision_transform(1.0f);

if (true == line_sphere_intersect(main_camera.eye, ray, vec3(0, 0, 0), 0.5f, t))
{
	collision_location = main_camera.eye + ray * t;
	col_loc = sphere;
	first_assignment = false;
	collision_transform = mat4(1.0f);
}

for (size_t i = 0; i < player_game_piece_meshes.size(); i++)
{
	glm::mat4 inverse = glm::inverse(player_game_piece_meshes[i].model_mat);
	glm::vec4 start = inverse * glm::vec4(main_camera.eye, 1.0);
	glm::vec4 direction = inverse * glm::vec4(ray, 0.0);
	direction = glm::normalize(direction);

	if (true == player_game_piece_meshes[i].intersect_AABB(start, direction))
	{
		vec3 closest_intersection_point;

		if (true == player_game_piece_meshes[i].intersect_triangles(start, direction, closest_intersection_point))
		{
			closest_intersection_point = player_game_piece_meshes[i].model_mat * vec4(closest_intersection_point, 1);

			if (first_assignment)
			{
				collision_location = closest_intersection_point;

				col_loc = player_game_piece;
				collision_location_index = i;

				first_assignment = false;
				collision_transform = inverse;
			}
			else
			{
				vec3 c0 = vec3(main_camera.eye) - closest_intersection_point;
				vec3 c1 = vec3(main_camera.eye) - collision_location;

				if (length(c0) < length(c1))
				{
					collision_location = closest_intersection_point;

					col_loc = player_game_piece;
					collision_location_index = i;
					collision_transform = inverse;
				}
			}
		}
	}
}


for (size_t i = 0; i < enemy_game_piece_meshes.size(); i++)
{
	glm::mat4 inverse = glm::inverse(enemy_game_piece_meshes[i].model_mat);
	glm::vec4 start = inverse * glm::vec4(main_camera.eye, 1.0);
	glm::vec4 direction = inverse * glm::vec4(ray, 0.0);
	direction = glm::normalize(direction);

	if (true == enemy_game_piece_meshes[i].intersect_AABB(start, direction))
	{
		vec3 closest_intersection_point;

		if (true == enemy_game_piece_meshes[i].intersect_triangles(start, direction, closest_intersection_point))
		{
			closest_intersection_point = enemy_game_piece_meshes[i].model_mat * vec4(closest_intersection_point, 1);

			if (first_assignment)
			{
				collision_location = closest_intersection_point;

				col_loc = enemy_game_piece;
				collision_location_index = i;

				first_assignment = false;
				collision_transform = inverse;
			}
			else
			{
				vec3 c0 = vec3(main_camera.eye) - closest_intersection_point;
				vec3 c1 = vec3(main_camera.eye) - collision_location;

				if (length(c0) < length(c1))
				{
					collision_location = closest_intersection_point;

					col_loc = enemy_game_piece;
					collision_location_index = i;
					collision_transform = inverse;
				}
			}
		}
	}
}

collision_location = collision_transform* vec4(collision_location, 1);

if (first_assignment)
{
	collision_location = vec3(0, 0, 0);
	col_loc = background;
}