I heard that an OpenGL projection matrix needs all 4x4 cells to store the values. An API I am using has 3x3 + vec3 representation of the projection matrix. I am just wondering how I am able to convert my 4x4 matrix to the API's format and back. This API is fully closed source so I have no idea what is going on in the background. I managed to convert the world matrix correctly but having no luck with the view/proj matrices as they produce incorrect visual results.

This is what I have so far:

[source]
CIwFMat ConvertToMarmaladeFormat(Matrix4& Mat)
{
CIwFMat M;
M.m[0][0] = Mat[0][0];
M.m[0][1] = Mat[0][1];
M.m[0][2] = Mat[0][2];
M.m[1][0] = Mat[1][0];
M.m[1][1] = Mat[1][1];
M.m[1][2] = Mat[1][2];
M.m[2][0] = Mat[2][0];
M.m[2][1] = Mat[2][1];
M.m[2][2] = Mat[2][2];
M.t = CIwFVec3(Mat[3][0], Mat[3][1], Mat[3][2]);
return M;
}

Matrix4 ConvertFromMarmaladeFormat(CIwFMat& M)
{
Matrix4 Mat;
Mat.identity();
Mat[0][0] = M.m[0][0];
Mat[0][1] = M.m[0][1];
Mat[0][2] = M.m[0][2];
Mat[1][0] = M.m[1][0];
Mat[1][1] = M.m[1][1];
Mat[1][2] = M.m[1][2];
Mat[2][0] = M.m[2][0];
Mat[2][1] = M.m[2][1];
Mat[2][2] = M.m[2][2];
Mat[3][0] = M.t.x;
Mat[3][1] = M.t.y;
Mat[3][2] = M.t.z;
return Mat;
}

class Matrix4
{
friend Vector4 operator*(const Vector4 &lhs, const Matrix4 &rhs);
friend Vector3 operator*(const Vector3 &lhs, const Matrix4 &rhs);
friend Matrix4 operator*(float scalar, const Matrix4 &rhs);
public:
static const Matrix4 IDENTITY;
static Matrix4 createFromAxes(const Vector3 &x, const Vector3 &y, const Vector3 &z);
static Matrix4 createFromAxesTransposed(const Vector3 &x, const Vector3 &y, const Vector3 &z);
static Matrix4 createFromHeadPitchRoll(float headDegrees, float pitchDegrees, float rollDegrees);
static Matrix4 createMirror(const Vector3 &planeNormal, const Vector3 &pointOnPlane);
static Matrix4 createOrient(const Vector3 &from, const Vector3 &to);
static Matrix4 createRotate(const Vector3 &axis, float degrees);
static Matrix4 createScale(float sx, float sy, float sz);
static Matrix4 createTranslate(float tx, float ty, float tz);
Matrix4() {}
Matrix4(float m11, float m12, float m13, float m14,
float m21, float m22, float m23, float m24,
float m31, float m32, float m33, float m34,
float m41, float m42, float m43, float m44);
~Matrix4() {}
float *operator[](int row);
const float *operator[](int row) const;
bool operator==(const Matrix4 &rhs) const;
bool operator!=(const Matrix4 &rhs) const;
Matrix4 &operator+=(const Matrix4 &rhs);
Matrix4 &operator-=(const Matrix4 &rhs);
Matrix4 &operator*=(const Matrix4 &rhs);
Matrix4 &operator*=(float scalar);
Matrix4 &operator/=(float scalar);
Matrix4 operator+(const Matrix4 &rhs) const;
Matrix4 operator-(const Matrix4 &rhs) const;
Matrix4 operator*(const Matrix4 &rhs) const;
Matrix4 operator*(float scalar) const;
Matrix4 operator/(float scalar) const;
float determinant() const;
void fromAxes(const Vector3 &x, const Vector3 &y, const Vector3 &z);
void fromAxesTransposed(const Vector3 &x, const Vector3 &y, const Vector3 &z);
void fromHeadPitchRoll(float headDegrees, float pitchDegrees, float rollDegrees);
void identity();
Matrix4 inverse() const;
void orient(const Vector3 &from, const Vector3 &to);
void rotate(const Vector3 &axis, float degrees);
void scale(float sx, float sy, float sz);
void toAxes(Vector3 &x, Vector3 &y, Vector3 &z) const;
void toAxesTransposed(Vector3 &x, Vector3 &y, Vector3 &z) const;
void toHeadPitchRoll(float &headDegrees, float &pitchDegrees, float &rollDegrees) const;
void translate(float tx, float ty, float tz);
Matrix4 transpose() const;
private:
float mtx[4][4];
};

class CIwFMat
{
public:
/**
* 3x3 rotation matrix.
*/
float m[3][3];
/**
* Trans vector.
*/
CIwFVec3 t;
...
};
[/source]

OpenGL uses 4x4 matrices for affine transformations, which are forced to have a last column (or row, depending on convention) identically (0 0 0 1). The last row indicates the translation and the remaining 3x3 matrix represents an endomorphism of the vector space. If the words I am using don't make sense, don't worry about it: I just mean that it makes sense to represent that as a 3x3 matrix and a vector. But projection matrices are different, and you may have to find out from the documentation of the API exactly what they use, and then construct the matrix appropriately.

Thanks for the information. There is nothing in the documentation specific to projection matrices and there is no 4x4 class in that API at all. Is there any suggestion you can give me to try and match my matrices given my situation? I have been stuck on this problem for about a week and ran out of ideas.

What does the prototype of a function that uses a projection look like? And what documentation do you have?

There are many combinations with a 3x3 matrix and vec3 that will yield a projection matrix, so I'm just guessing here.
It's a long shot, but why won't you give it a try. You initialize your matrix to identity before putting stuff into it. Still do that, but also put zero into [3][3] and put -1 into [2][3] (if -1 doesn't work, try 1).