C# snippet:

public static class BoxOutline
{
	private   static readonly int []   OUTLINE = new int [43]
	                                   {
	                                       0x00000000, // 00 ... inside
	                                       0x00003F04, // 01 ... left ................... 0,4,7,3
	                                       0x00005C8C, // 02 ... right .................. 1,2,6,5
	                                       0x00000000, // 03 
	                                       0x00004A44, // 04 ... bottom ................. 0,1,5,4
	                                       0x000FCA46, // 05 ... bottom, left ........... 0,1,5,4,7,3
	                                       0x0012E446, // 06 ... bottom, right .......... 0,1,2,6,5,4
	                                       0x00000000, // 07 
	                                       0x00006ED4, // 08 ... top .................... 2,3,7,6
	                                       0x0001ADE6, // 09 ... top, left .............. 4,7,6,2,3,0
	                                       0x0006EED6, // 10 ... top, right ............. 2,3,7,6,5,1
	                                       0x00000000, // 11 
	                                       0x00000000, // 12 
	                                       0x00000000, // 13 
	                                       0x00000000, // 14 
	                                       0x00000000, // 15 
	                                       0x000014C4, // 16 ... front .................. 0,3,2,1
	                                       0x000FC056, // 17 ... front, left ............ 2,1,0,4,7,3
	                                       0x0006E4C6, // 18 ... front, right ........... 0,3,2,6,5,1
	                                       0x00000000, // 19 
	                                       0x001294C6, // 20 ... front, bottom .......... 0,3,2,1,5,4
	                                       0x000FCA56, // 21 ... front, bottom, left .... 2,1,5,4,7,3
	                                       0x0012E4C6, // 22 ... front, bottom, right ... 0,3,2,6,5,4
	                                       0x00000000, // 23 
	                                       0x00056EC6, // 24 ... front, top ............. 0,3,7,6,2,1
	                                       0x00056F06, // 25 ... front, top, left ....... 0,4,7,6,2,1
	                                       0x0006EEC6, // 26 ... front, top, right ...... 0,3,7,6,5,1
	                                       0x00000000, // 27 
	                                       0x00000000, // 28 
	                                       0x00000000, // 29 
	                                       0x00000000, // 30 
	                                       0x00000000, // 31 
	                                       0x00007D64, // 32 ... back ................... 4,5,6,7
	                                       0x0001FD66, // 33 ... back, left ............. 4,5,6,7,3,0
	                                       0x00167C8E, // 34 ... back, right ............ 1,2,6,7,4,5
	                                       0x00000000, // 35 
	                                       0x0013EA46, // 36 ... back, bottom ........... 0,1,5,6,7,4
	                                       0x000FEA46, // 37 ... back, bottom, left ..... 0,1,5,6,7,3
	                                       0x0013E446, // 38 ... back, bottom, right .... 0,1,2,6,7,4
	                                       0x00000000, // 39 
	                                       0x001ACED6, // 40 ... back, top .............. 2,3,7,4,5,6
	                                       0x000D6B06, // 41 ... back, top, left ........ 0,4,5,6,2,3
	                                       0x0016768E  // 42 ... back, top, right ....... 1,2,3,7,4,5
	                                   };
	
	private   static readonly Vector3 []   m_Corners = new Vector3[8];
	
	//-----------------------------------------------------------------------------------------------------------------
	static public int GetOutline( Vector3 boxMin, Vector3 boxMax, Vector3 camPos, Vector3 [/*6*/] outline )
	{
		int idx = ( camPos.x < boxMin.x ?  1 : 0 )   //  1 = left
		        + ( camPos.x > boxMax.x ?  2 : 0 )   //  2 = right
		        + ( camPos.y < boxMin.y ?  4 : 0 )   //  4 = bottom
		        + ( camPos.y > boxMax.y ?  8 : 0 )   //  8 = top
		        + ( camPos.z < boxMin.z ? 16 : 0 )   // 16 = front
		        + ( camPos.z > boxMax.z ? 32 : 0 );  // 32 = back
		
		if ( idx >= OUTLINE.Length )
		return 0;
		
		int enc = OUTLINE[ idx ];
		
		if ( enc == 0 )
		return 0;
		
		GetCorners( boxMin, boxMax, m_Corners );
		
		int num = enc & 7;                                             // decodes count of vertices in outline
		
		for ( idx = 0; idx < num; ++idx )
		{
			enc >>= 3;   outline[ idx ] = m_Corners[ enc & 7 ];    // decodes outline vertices one by one
		}
		
		return num;
	}
	
	//-----------------------------------------------------------------------------------------------------------------
	public static void GetCorners( Vector3 boxMin, Vector3 boxMax, Vector3 [/*8*/] corners )
	{
		corners[ 0 ] = new Vector3( boxMin.x, boxMin.y, boxMin.z );   // 3--------------2
		corners[ 1 ] = new Vector3( boxMax.x, boxMin.y, boxMin.z );   // | \          / |
		corners[ 2 ] = new Vector3( boxMax.x, boxMax.y, boxMin.z );   // |  7 ------ 6  |
		corners[ 3 ] = new Vector3( boxMin.x, boxMax.y, boxMin.z );   // |  |        |  |
		                                                              // |  |        |  |
		corners[ 4 ] = new Vector3( boxMin.x, boxMin.y, boxMax.z );   // |  |        |  |
		corners[ 5 ] = new Vector3( boxMax.x, boxMin.y, boxMax.z );   // |  4 ------ 5  |
		corners[ 6 ] = new Vector3( boxMax.x, boxMax.y, boxMax.z );   // | /          \ |
		corners[ 7 ] = new Vector3( boxMin.x, boxMax.y, boxMax.z );   // 0 -------------1
	}
}

EDIT 2015-07-14 : simplified

EDIT 2015-07-17 : added comments so it is clear how it works :)

What I'd do is transform all the 8 corner points of the cube to 2D screen space, and then use a 2D convex hull algorithm to recompute which edges are the outside edges of the projected silhouette. That solution has the appeal of using only generic "stock" algorithms (3D->2D projection, and 2D convex hull), so it will be very easy to reason about and maintain in the long run.

Hmm, Id have to research a bit on that, I'm not entirely sure what you mean but it sounds interesting. I'll look up what stock algo's are and the approach with a 2D convex hull. Thanks a lot!

untested C# snippet:

public static class BoxOutline
{
	private static readonly   Vector3 []   CORNERS = { new Vector3( -0.5f, -0.5f, -0.5f ),
	                                                   new Vector3(  0.5f, -0.5f, -0.5f ),
	                                                   new Vector3(  0.5f,  0.5f, -0.5f ), 
	                                                   new Vector3( -0.5f,  0.5f, -0.5f ),
	                                                   new Vector3( -0.5f, -0.5f,  0.5f ), 
	                                                   new Vector3(  0.5f, -0.5f,  0.5f ),
	                                                   new Vector3(  0.5f,  0.5f,  0.5f ), 
	                                                   new Vector3( -0.5f,  0.5f,  0.5f ) };
	
	private static readonly   int []       OUTLINE = new int [43] { 0x00000000,
	                                                                0x00003F04,
	                                                                0x00005C8C,
	                                                                0x00000000,
	                                                                0x00004A44,
	                                                                0x000FCA46,
	                                                                0x0012E446,
	                                                                0x00000000,
	                                                                0x00006ED4,
	                                                                0x0001ADE6,
	                                                                0x0006EED6,
	                                                                0x00000000,
	                                                                0x00000000,
	                                                                0x00000000,
	                                                                0x00000000,
	                                                                0x00000000,
	                                                                0x000014C4,
	                                                                0x000FCA56,
	                                                                0x0012E4C6,
	                                                                0x00000000,
	                                                                0x001294C6,
	                                                                0x000FCA56,
	                                                                0x0012E4C6,
	                                                                0x00000000,
	                                                                0x00056EC6,
	                                                                0x00056F06,
	                                                                0x0006EEC6,
	                                                                0x00000000,
	                                                                0x00000000,
	                                                                0x00000000,
	                                                                0x00000000,
	                                                                0x00000000,
	                                                                0x00007D64,
	                                                                0x0001FD66,
	                                                                0x00167C8E,
	                                                                0x00000000,
	                                                                0x0013EA46,
	                                                                0x000FEA46,
	                                                                0x0013E446,
	                                                                0x00000000,
	                                                                0x001ACED6,
	                                                                0x000D6B06,
	                                                                0x0016768E };
	
	private   static Vector3 []   m_Corners = new Vector3[8];
	
	//-----------------------------------------------------------------------------------------------------------------
	static public int GetOutline( Vector3 boxMin, Vector3 boxMax, Vector3 camPos, Vector3 [/*6*/] outline )
	{
		int idx = ( camPos.x < boxMin.x ?  1 : 0 )
		        + ( camPos.x > boxMax.x ?  2 : 0 )
		        + ( camPos.y < boxMin.y ?  4 : 0 )
		        + ( camPos.y > boxMax.y ?  8 : 0 )
		        + ( camPos.z < boxMin.z ? 16 : 0 )
		        + ( camPos.z > boxMax.z ? 32 : 0 );
		
		if ( idx >= OUTLINE.Length )
		return 0;
		
		int enc = OUTLINE[ idx ];
		
		if ( enc == 0 )
		return 0;
		
		GetCorners( boxMin, boxMax, m_Corners );
		
		int num = enc & 7;
		
		for ( idx = 0; idx < num; ++idx )
		{
			enc >>= 3;   outline[ idx ] = m_Corners[ enc & 7 ];
		}
		
		return num;
	}
	
	//-----------------------------------------------------------------------------------------------------------------
	public static void GetCorners( Vector3 boxMin, Vector3 boxMax, Vector3 [/*8*/] corners )
	{
		Vector3  center = ( boxMin + boxMax ) * 0.5f;
		Vector3  size   = ( boxMax - boxMin );
		
		for ( int i = 0; i < 8; ++i )
		{
			corners[i] = center + Vector3.Scale( size, CORNERS[i] );
		}
	}
}

Man that looks a lot simpler than my code but I haven't tested it out yet. Will do in just one second. I will for sure have questions on how some of this works.

I encountered an error with this approach in general. My idea for getting the outer edges of a cube was so that I could render octrees and figure out the outline for the root and which points of the octrees root are in it (and their order). When I subdivide my tree, I would render the children but use the same points as I did for the root to generate an outline for them and while my outline is correct, I got into this error:

It may not be obvious what is happening at first so Ill single out 2 nodes and render its 6 visible vertices.

Notice how different faces are visible for these 2 nodes. For the node on the left, only 2 faces are visible while for the node on the right, 3 faces are visible. Here is an illustration(the perspective is exaggerated):

I'm a little confused on how to approach this. I do not want to find the outline on a per node basis since I can expect at least a million traversals a frame. Whats more is the paper which i'm trying to replicate (http://www.cs.princeton.edu/courses/archive/spr01/cs598b/papers/greene96.pdf) does the same approach where you only render the outline and I doubt they would find the outline on a per node basis given their performance (it would seem like way too large of a draw back). I'm thinking that there may be a way of tweaking visible faces given the cubes screen position but the cube shares the exact same orientation as its root so it would seem that if only 2 nodes for the root are visible then only 2 are visible for all the children as well.

EDIT: And this is what the root node looks like for the above example

Method 1 (easy one): If the model is convex (if not refer to method 2) you project each edge to 2D space (its not that much of a projection, just ignore the z value). then only those edges are wanted that are on only one side of the other vertices (meaning vertices shouldn't exist one both sides of the wanted edge) to do this if we have ax+by+c=0 as the edges line equation, and our vertices be {(x₁,y₁),...,(x_n,y_n)}, for y=ax_i+by_i+c we shouldn't have both negative and positive values.

Method 2: for each of vertices of the edge we for all polygons check if the vertices are inside of the polygon. if they weren't inside any of the polygons, that edge is chosen to be drawn.

and after choosing the edges it's a simple sort off the edges to find the order of drawing them.

I fixed my snippet ("front left", "front right" cases) and made it little shorter :).

I fixed my snippet ("front left", "front right" cases) and made it little shorter .

For your snippet, is box min and box max the corners of the cube? Are they expected to be axis alligned (an AABB)? I'm attempting to implement your snippet and get it working but I'm not sure what space your function is assuming. It seems like its world space but when I implement it, it doesn't generate proper values:

Mat4x4f MatPos = SetWorldPos(Pos);
	Mat4x4f MatRot = SetRotation(Rot);
	Mat4x4f MatScale = SetScale(Scale);
	Mat4x4f MatCamera = SetCamera(Camera);
	Mat4x4f MatProject = SetProjection(Screen, 90.0f, 0.01f, 100.0f);
	Mat4x4f Transform = MatCamera * MatPos * MatRot * MatScale;

	Vector3f Outline[6];
	int32 len = GetOutline(MatPos * MatRot * MatScale * Vector3f{ -0.5f, -0.5f, -0.5f },
		MatPos * MatRot * MatScale * Vector3f{ 0.5f, 0.5f, 0.5f }, Camera->Pos, Outline);
	for (int32 i = 0; i < 6; ++i)
	{
		Outline[i] = ProjectToScreen(Screen, MatProject * MatCamera * Outline[i]); // Converts from NDC to screen coords
	}
	RenderCubeOutline(Screen, Outline[0], Outline[1], Outline[2], Outline[3], Outline[4], Outline[5], 0xFFFFFFFF);

Parameters 'boxMin' and 'boxMax' are "minimum" and "maximum" of axis-aligned box (AABB). "Straightforward" usage would be for AABBs in world-space but if you transform your camera in local space of box then it can be used for oriented boxes (OBB) too. These still can be some bugs in those hex constants... I was testing it at most 2 mins so... :)

Parameters 'boxMin' and 'boxMax' are "minimum" and "maximum" of axis-aligned box (AABB). "Straightforward" usage would be for AABBs in world-space but if you transform your camera in local space of box then it can be used for oriented boxes (OBB) too. These still can be some bugs in those hex constants... I was testing it at most 2 mins so... :)

Ah makes sense. Im not entirely sure what your masks represent. Could you give some more insight? It looks really interesting. I probably made a mistake with my math, ill look over how i call it.

Does anyone have any ideas to my post about how the outline isn't always the same and cant be propogated down the octree traversal? Is my only option to calcukate it until my nodes are in one of the 4 screen quadrants?

I edited my snippet again.... each bit-mask (for given situation) contains number of vertices in outline (4 or 6, stored in lowest 3 bits)

followed by corresponding indices (0-7, each again stored in 3 bits).

This approach should be easily extendable (from "vertices-on-outline") to "front-facing-quads" or "front-facing-triangle-strips", etc.

This approach should be easily extendable (from "vertices-on-outline") to "front-facing-quads" or "front-facing-triangle-strips", etc.

But it can't be propogated down an octree. For example, the vertices that make up the outline for the root of the octree are not the same vertices that make up the outline for the children of that root. Unfortunatly, it looks like i have to recalculate the outline on each traversal until my cube is in either the top left, top right, bot left, bot right (can't be in 2 parts at the same time). Unfortunatly this is probably going to be a lot more performance intensive but ill give it an implementation.

Also, i dont want to approximate a cube with a quad that encompases in screenspace because im tryingn to minimize my traversals and using quads covers more area than the object actually takes up (creating many false positives).

Sorry for leaving for a while, had some personal stuff and work eat up all my time.