Bilinear Texture Filtering – Artifacts, Alternatives, and Frequency Domain Analysis

Bilinear texture filtering – artifacts, alternatives, and frequency domain analysis | Bart Wronski var wpcom_remote_login_extra_auth = ''; function wpcom_remote_login_remove_dom_node_id( element_id ) { var dom_node = document.getElementById( element_id ); if ( dom_node ) { dom_node.remove(); } } function wpcom_remote_login_remove_dom_node_classes( class_name ) { var dom_nodes = document.querySelectorAll( '.' + class_name ); for ( var i = 0; i < dom_nodes.length; i++ ) { dom_nodes[i].remove(); } } function wpcom_remote_login_final_cleanup() { wpcom_remote_login_remove_dom_node_classes( "wpcom_remote_login_msg" ); wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_key" ); wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_validate" ); wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_js" ); wpcom_remote_login_remove_dom_node_id( "wpcom_request_access_iframe" ); wpcom_remote_login_remove_dom_node_id( "wpcom_request_access_styles" ); } // Watch for messages back from the remote login window.addEventListener( "message", function( e ) { if ( e.origin === "https://r-login.wordpress.com" ) { var data = {}; try { data = JSON.parse( e.data ); } catch( e ) { wpcom_remote_login_final_cleanup(); return; } if ( data.msg === 'LOGIN' ) { // Clean up the login check iframe wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_key" ); var id_regex = new RegExp( /^[0-9]+$/ ); var token_regex = new RegExp( /^.*|.*|.*$/ ); if ( token_regex.test( data.token ) && id_regex.test( data.wpcomid ) ) { // We have everything we need to ask for a login var script = document.createElement( "script" ); script.setAttribute( "id", "wpcom_remote_login_validate" ); script.src = '/remote-login.php?wpcom_remote_login=validate' + '&wpcomid=' + data.wpcomid + '&token=' + encodeURIComponent( data.token ) + '&host=' + window.location.protocol + '//' + window.location.hostname; document.body.appendChild( script ); } return; } // Safari ITP, not logged in, so redirect if ( data.msg === 'LOGIN-REDIRECT' ) { window.location = 'https://wordpress.com/log-in?redirect_to=' + window.location.href; return; } // Safari ITP, storage access failed, remove the request if ( data.msg === 'LOGIN-REMOVE' ) { var css_zap = 'html { -webkit-transition: margin-top 1s; transition: margin-top 1s; } /* 9001 */ html { margin-top: 0 !important; } * html body { margin-top: 0 !important; } @media screen and ( max-width: 782px ) { html { margin-top: 0 !important; } * html body { margin-top: 0 !important; } }'; var style_zap = document.createElement( 'style' ); style_zap.type = 'text/css'; style_zap.appendChild( document.createTextNode( css_zap ) ); document.body.appendChild( style_zap ); var e = document.getElementById( 'wpcom_request_access_iframe' ); e.parentNode.removeChild( e ); document.cookie = 'wordpress_com_login_access=denied; path=/; max-age=31536000'; return; } // Safari ITP if ( data.msg === 'REQUEST_ACCESS' ) { console.log( 'request access: safari' ); // Check ITP iframe enable/disable knob if ( wpcom_remote_login_extra_auth !== 'safari_itp_iframe' ) { return; } // If we are in a "private window" there is no ITP. var private_window = false; try { var opendb = window.openDatabase( null, null, null, null ); } catch( e ) { private_window = true; } if ( private_window ) { console.log( 'private window' ); return; } var iframe = document.createElement( 'iframe' ); iframe.id = 'wpcom_request_access_iframe'; iframe.setAttribute( 'scrolling', 'no' ); iframe.setAttribute( 'sandbox', 'allow-storage-access-by-user-activation allow-scripts allow-same-origin allow-top-navigation-by-user-activation' ); iframe.src = 'https://r-login.wordpress.com/remote-login.php?wpcom_remote_login=request_access&origin=' + encodeURIComponent( data.origin ) + '&wpcomid=' + encodeURIComponent( data.wpcomid ); var css = 'html { -webkit-transition: margin-top 1s; transition: margin-top 1s; } /* 9001 */ html { margin-top: 46px !important; } * html body { margin-top: 46px !important; } @media screen and ( max-width: 660px ) { html { margin-top: 71px !important; } * html body { margin-top: 71px !important; } #wpcom_request_access_iframe { display: block; height: 71px !important; } } #wpcom_request_access_iframe { border: 0px; height: 46px; position: fixed; top: 0; left: 0; width: 100%; min-width: 100%; z-index: 99999; background: #23282d; } '; var style = document.createElement( 'style' ); style.type = 'text/css'; style.id = 'wpcom_request_access_styles'; style.appendChild( document.createTextNode( css ) ); document.body.appendChild( style ); document.body.appendChild( iframe ); } if ( data.msg === 'DONE' ) { wpcom_remote_login_final_cleanup(); } } }, false ); // Inject the remote login iframe after the page has had a chance to load // more critical resources window.addEventListener( "DOMContentLoaded", function( e ) { var iframe = document.createElement( "iframe" ); iframe.style.display = "none"; iframe.setAttribute( "scrolling", "no" ); iframe.setAttribute( "id", "wpcom_remote_login_key" ); iframe.src = "https://r-login.wordpress.com/remote-login.php" + "?wpcom_remote_login=key" + "&origin=aHR0cHM6Ly9iYXJ0d3JvbnNraS5jb20%3D" + "&wpcomid=62253954" + "&time=1587423149"; document.body.appendChild( iframe ); }, false ); /* */ window._wpemojiSettings = {"baseUrl":"https:\/\/s0.wp.com\/wp-content\/mu-plugins\/wpcom-smileys\/twemoji\/2\/72x72\/","ext":".png","svgUrl":"https:\/\/s0.wp.com\/wp-content\/mu-plugins\/wpcom-smileys\/twemoji\/2\/svg\/","svgExt":".svg","source":{"concatemoji":"https:\/\/s0.wp.com\/wp-includes\/js\/wp-emoji-release.min.js?m=1582709031h&ver=5.4"}}; /*! This file is auto-generated */ !function(e,a,t){var r,n,o,i,p=a.createElement("canvas"),s=p.getContext&&p.getContext("2d");function c(e,t){var a=String.fromCharCode;s.clearRect(0,0,p.width,p.height),s.fillText(a.apply(this,e),0,0);var r=p.toDataURL();return s.clearRect(0,0,p.width,p.height),s.fillText(a.apply(this,t),0,0),r===p.toDataURL()}function l(e){if(!s||!s.fillText)return!1;switch(s.textBaseline="top",s.font="600 32px Arial",e){case"flag":return!c([127987,65039,8205,9895,65039],[127987,65039,8203,9895,65039])&&(!c([55356,56826,55356,56819],[55356,56826,8203,55356,56819])&&!c([55356,57332,56128,56423,56128,56418,56128,56421,56128,56430,56128,56423,56128,56447],[55356,57332,8203,56128,56423,8203,56128,56418,8203,56128,56421,8203,56128,56430,8203,56128,56423,8203,56128,56447]));case"emoji":return!c([55357,56424,55356,57342,8205,55358,56605,8205,55357,56424,55356,57340],[55357,56424,55356,57342,8203,55358,56605,8203,55357,56424,55356,57340])}return!1}function d(e){var t=a.createElement("script");t.src=e,t.defer=t.type="text/javascript",a.getElementsByTagName("head")[0].appendChild(t)}for(i=Array("flag","emoji"),t.supports={everything:!0,everythingExceptFlag:!0},o=0;o img.wp-smiley, img.emoji { display: inline !important; border: none !important; box-shadow: none !important; height: 1em !important; width: 1em !important; margin: 0 .07em !important; vertical-align: -0.1em !important; background: none !important; padding: 0 !important; } .has-text-align-justify { text-align:justify; } :root { --font-headings: unset; --font-base: unset; --font-headings-default: -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Oxygen-Sans,Ubuntu,Cantarell,"Helvetica Neue",sans-serif; --font-base-default: -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Oxygen-Sans,Ubuntu,Cantarell,"Helvetica Neue",sans-serif;} .recentcomments a { display: inline !important; padding: 0 !important; margin: 0 !important; } table.recentcommentsavatartop img.avatar, table.recentcommentsavatarend img.avatar { border: 0px; margin: 0; } table.recentcommentsavatartop a, table.recentcommentsavatarend a { border: 0px !important; background-color: transparent !important; } td.recentcommentsavatarend, td.recentcommentsavatartop { padding: 0px 0px 1px 0px; margin: 0px; } td.recentcommentstextend { border: none !important; padding: 0px 0px 2px 10px; } .rtl td.recentcommentstextend { padding: 0px 10px 2px 0px; } td.recentcommentstexttop { border: none; padding: 0px 0px 0px 10px; } .rtl td.recentcommentstexttop { padding: 0px 10px 0px 0px; } body.custom-background { background-color: #AAB3AB; } #access ul li.current_page_item>a,#access ul li.current_page_parent>a,#access ul li.current-menu-ancestor>a,#access ul li.current-menu-item>a,#access ul li.current-menu-parent>a{color:#5b5b5b}#access li:hover>a,#access ul ul *:hover>a{fg2:#fff}#access a{color:#515151}.entry-meta a:link,.entry-utility a:link{color:#888}body{background-color:#aab3ab}#access,#access ul ul a{background-color:#eee0b7}#access li:hover>a,#access ul ul *:hover>a{background-color:#e8caaf}a:link,a:visited{color:#657155}.home .sticky{background-color:#c4cbb7}.home .sticky{background-color:rgba(196,203,183,.05)}.home .sticky .page-link a{background-color:#c4cbb7}.home .sticky .page-link a{background-color:rgba(196,203,183,.2)}a:hover,a:focus,a:active{color:#69712c}.entry-title a:active,.entry-title a:hover{color:#69712c}.page-title a:active,.page-title a:hover{color:#69712c}.entry-meta a:hover,.entry-utility a:hover{color:#69712c}.navigation a:active,.navigation a:hover{color:#69712c}.comment-meta a:active,.comment-meta a:hover{color:#69712c}.reply a:hover,a.comment-edit-link:hover{color:#69712c}.page-link a:active,.page-link a:hover{color:#69712c} Bart Wronski Graphics, machine learning, image and signal processing Skip to content Home Articles / blog posts index Publications Personal projects About me ← Using JAX, numpy, and optimization techniques to improve separable image filters Bilinear texture filtering – artifacts, alternatives, and frequency domain analysis Posted on April 14, 2020 by bartwronski In this post we will look at one of the staples of real-time computer graphics – bilinear texture filtering. To catch your interest, I will start with focusing on something that is often referred to as “bilinear artifacts”, trapezoid/star-shaped artifact of bilinear interpolation – what causes them? I will discuss briefly some common bilinear filtering alternatives and how they fix those, link a few of my favorite papers on (fast) image interpolation, and analyze the frequency response of common cheap filters. Screenshot from my favorite video game of all time – Deus Ex (2000). See this diamond shape of the pupil? This is what I will refer to as “bilinear artifacts”. As usual, I will work on the topic in “layers”, starting with basics and “introduction to graphics” level and perspective – analyzing the “spatial” side of the artifact. Then I will (non-exhaustively) the alternatives – bicubic and biquadratic filter, and finally analyze those from a signal processing / EE / Fourier spectrum perspective. I will also comment on relationship between filtering as used in the context of upsampling / interpolation, and shifting images. (Note: I am going to completely ignore here downsampling, decimation, and generally trilinear filtering / mip-mapping.) What is bilinear filtering and why graphics use it so much? I assume that most of the audience of my blog post are graphics practitioners or enthusiasts and most of us use bilinear filtering every day, but a quick recap never hurts. Bilinear filtering is a texture (or more generally, signal) interpolation filter that is separable – it is a linear filter applied on the x axis of the image (along the width), and then a second filter applied along the y axis (along the height). Note on notation: Throughout this post I will use linear/bilinear almost interchangeably due to this property of bilinear filtering being a linear filter applied in the two directions sequentially. Starting with a single dimension, linear filter is often called a “tent” or “triangle” filter. In the context of input texture interpolation, we can look at it as either “gather” (“when computing an output pixel, which samples contribute to it?”), or “scatter” operation (“for each input sample, what are its contributions to the final result?”). Most of GPU programmers naturally think of “gather” operations, but here this “triangle” part is important, so let’s start with the scatter. For every input pixel, its contributions to the output signal form a “tent” – a “triangle” of pixel contributions is centered at sample N, with height equal to the sample/pixel N value, and its base spans from the sample (N-1) to (N+1). To get the filtering result we sum all contributing “tents” (there are either 1 or 2 covering each output pixel). All of triangles have the same base length, and all of them overlap with a single other triangle on either side of the top vertex. Here is a diagram that hopefully will help: (Bi)linear filtering is called triangle or tent filtering as contributions of the input samples to the filtered results are “triangles” or “tents”. Switching from “scatter” to a more natural (at least to people who write shaders) gather approach – between samples (N+1) and (N) we sum up contributions of all “tents” covering this area, so (N+1) and N, and at exact position of the sample N we get only its contribution. Formula for this line is simply xn * (t – 1) + xn-1 * t. This means that in 1D each output sample has two contributing input pixels (though sometimes one contribution is zero!), and we linearly interpolate between them. If we repeat this twice per x and y, we end up with four input input samples and four weights (t_x – 1) * (t_y – 1), (t_x ) * (t_y – 1), (t_x – 1) * (t_y ), (t_x) * (t_y). Since the weights are just multiplication of independent x and y weights, we can verify that this filter is separable (if you are still unsure what it means in practice, check out my previous blog post on separable filters). Bilinear filtering is ubiquitous in graphics and there are a few reasons for it – but the main one is that is super cheap, and furthermore, hardware accelerated; and that it provides significant quality improvement over nearest-neighbor filtering. All modern GPUs can do bilinear filtering of an input texture in a single hardware instruction. This involves both memory access (together with potentially uncompressing block compressed textures into local cache), as well as the interpolation arithmetic. Sometimes (depending on the architecture and input texture bit depth) the instruction might have slightly higher latency, but unless someone is micro-optimizing or has a very specific workload, the cost of the bilinear filtering can be considered “free” on contemporary hardware. At the same time, it provides significant quality improvement over “nearest neighbor” filtering (just replicating each sample N times) for low resolution textures. Nearest neighbor filtering (red) produces discontinuities, while bilinear sampling (blue) produces continuous values between samples. Bilinear vs nearest neighbor filtering. Being supported natively in hardware is a huge deal and the (bilinear) texture filtering was one of the main initial features of graphics accelerators (when they were still mostly separate cards, in addition to the actual GPUs). Personal part of the story – I still remember the day when I saw a few “demo” games on 3dfx Voodoo accelerator on my colleague’s machine (it was a Diamond Monster3D) back in primary school. Those buttery smooth textures and no pixels jumping around made me ask my father over and over again to get us one (at that time we had a single computer at home, and it was my father’s primary work tool) – a dream that came true a bit later, with Voodoo 2. Side remark – pixels vs filtering, interpolation, reconstruction One thing that is worth mentioning is that I deliberately don’t want to draw input – or even the output pixels as “little squares” – pixels are not little squares! Classic tech memo from Alvy Ray Smith elaborates on it, but if you want to to deeper into understanding filtering, I find it extremely important to not make such a “mistake” (I blame “nearest neighbor” interpolation for it). So what are those values stored in your texture maps and framebuffers? They are infinitely small samples taken at a given location. Between them, there is no signal at all – you can think of your image as “impulses”. Such signal is not very useful on its own, it needs a reconstruction filter that reconstructs a continuous, non discrete representation. Such a reconstruction filter can be your monitor and in this case, pixels could be actually squares (though different LCD monitors have different pixel patterns), or a blurred dot of a diffracted ray in CRT monitor. (This is something that contemporary pixel art gets wrong and Timothy Lottes used to have interesting blog posts about – I think they might have gotten deleted, but his shadertoy showing more correct emulation of a CRT monitor is still there). For me it is also useful to think of texture filtering / interpolation in a similar way – we are reconstructing a continuous (as in – not discrete/sampled) signal and then resample it, taking measurements / values at new pixel positions from this continuous representation. This idea was one of key components in our handheld mobile super-resolution work at Google, allowing to resample reconstructed signals at any desired magnification level. Example from published past work that I worked on on how given a continuous reconstruction of a sparse signal (from many locally randomly shifted input frames) can be sampled at different destination grid densities, achieving higher zoom factors with less aliasing and more details. Bilinear artifacts – spatial perspective – pyramid, and mach bands Ok, bilinear filtering seems to be “pretty good” and for sure is cheap! So what are those bilinear artifacts? Again a zoom-in into a screenshot from Deus Ex intro – notice the star-like pupil. In the post intro I have shown example artifact from a 20 year old video game back when the asset textures were very low resolution, but those happen anytime we interpolate from low resolution buffers. In my Siggraph 2014 talk about volumetric atmospheric effects rendering, I have shown an example of problematic artifacts when upsampling low resolution 3D fog texture: Volumetric fog bilinear artifacts – notice stair/saw-like look. In my old talk proposed to solve this problem with additional temporal low-pass effect (temporal jitter + temporal supersampling), because the alternative (bicubic / B-spline sampling – more about it later) was too expensive to apply in real time. Nota bene such a solution is actually pretty good at approximating blurring / filtering for free if you already have a TAA-like framework. If we get back to our “tent” filter scatter interpretation, it should become easier to see what causes this effect. Separable filtering applies first a 1D triangle filter in one direction, then in another direction, multiplying the weights. Those two 1D ramps multiplied together result in a pyramid – this means that every input pixel will “splat” a small pyramid, and if the ratio of the output to the input resolutions is large and textures have high contrast, then those will become very apparent. Similarly those pyramids on any edge that is not aligned with perfect 45 degrees, will create jaggy, aliased appearance. Horizontal tent filter followed by a vertical tent filter results in a “pyramid”. Ok, it’s supposed to be a pyramid, but what’s the deal with this bright star-like lines? The second “spatial” effect of why bilinear filtering is not great are so-called Mach bands. This phenomenon is one of the fascinating “artifacts” (or conversely – desired features / capabilities) of human visual system. Human vision cannot be thought of as a “sensor” like in a digital camera or “film” in analog one – it is more like a complicated video processing system, will all sorts of edge detectors, contrast boosting, motion detection, embedded localized tonemapping etc. Mach bands are caused by tendency of HVS to “sharpen” image and emphasize any discontinuity. Perfectly linear gradient (for example generated by linear interpolation). Do you also see this apparent bright white line on the right side of the gradient? (Bi)linear interpolation is continuous, however its derivative is not (derivative of a piecewise linear function is a piecewise constant function). In mathematical definition of smoothness and C-continuity we say that it is C0 continuous, but not C1 continuous. I find it fascinating that HVS can detect such discontinuity – detecting features like lines and corners is like differentiation and indeed, most common and basic feature detector in computer vision, Harris Corner Detector analyzes local gradient fields. To get rid of this effect, we need some filtering and interpolation that has a higher order of continuity. Before I move to some other filters, I have to mention here also a very smart “hack” from Inigo Quilez – by hacking the interpolation coordinates, you can get interpolation weights that are C1 continuous. Screenshot from: https://www.shadertoy.com/view/XsfGDn, author Inigo Quilez. Using hacked UVs together with hardware bilinear interpolation to get smoother filtering. Be sure to check Inigo’s excellent article. It’s a really cool hack and works very well with procedural art (where manually computed gradients are used to produce for example normal maps), but produces “rounded rectangles” type of look and aliases in motion, so let’s investigate some other fixes. Bilinear alternatives – bicubic / biquadratic The most common alternative considered in graphics is bicubic interpolation. The name itself is in my opinion confusing, as cubic filtering can mean any filtering where we use 4 samples, and filter weights result from evaluating 3rd order polynomials (similarly to linear using 2 samples and a 1st order polynomial). In 2D, we have 4×4 = 16 samples, but we also interpolate separably. However there are many alternative 3rd order polynomials weights that could be used for filtering… This is why I avoid using just the term “bicubic”. Just look at Photoshop image resizing options, does this make any sense!? Bicubic, or bicubic sharper? ¯\_(ツ)_/¯ The most common bicubic filter used in graphics is one that is also called B-Spline bicubic (after reading this post, see this classic paper from Don Mitchell and Arun Netravali that analyzes some other ones – I revisit this paper probably once every two months!). It looks like this: As you can see, it is way smoother and seems to reconstruct shapes much more faithfully than bilinear! Smoothness and lack of artifacts comes from its higher order continuity – it is designed to have matching and continuous derivatives at each original sample point. It has a few other cool and useful properties, but my personal favorite one is that because all weights are positive, it can be optimized to use just 4 bilinear samples! This old (but not dated!) article from GPU Gems 2 (and a later blog post from Phill Djonov elaborating on the derivation) describe how to do it with just some extra ALU to compute the look-up UVs. (Similarly in 1D this can be done in just 2 samples – which can be quite useful for low-dimensionality LUTs). Its biggest disadvantage – it is visibly “soft”/blurry, looks like the whole image got slightly blurred – which is what is happening… Some other cubic interpolators are sharper, but you have to pay a price for that – negative filter weights that can cause halos / ringing, as well as make it impossible to optimize so nicely. Second alternative that caught my attention and was actually a motivation to finally write something about image resampling was a recent blog post from Thomas Deliot (and an efficient shadertoy implementation from Leonard Ritter) on biquadratic interpolation – continuous interpolation that reconstructs a quadratic function and considers 3 input samples (or 9 in 2d). I won’t take the spotlight from them on how it works, I recommend checking the linked materials, but included an animated comparison – it looks very good, sharper than the bspline bicubic, and is a bit cheaper. Post update: Won Chun pointed to me a super interesting piece of literature on quadratic filters from Neil Dogson. The link is paywalled, but you will easily find references by googling for “Quadratic interpolation for image resampling”. It derives the quadratic filter with an additional parameter that allows to control filter sharpness similarly to bicubic filters parameters B/C. The one that I used for analysis is the blurriest, “approximating” one. Here is a shadertoy from Won Chun and a corresponding Desmos calculator that proposes also a 2 sample approximation of the (bi)quadratic filter, cool stuff! Interpolation – shifting reconstructed signal and position dependent filtering A second perspective that I am going to discuss here is how texture filtering creates different local filters and filter weights depending on the fractional output/input pixel positions relationship. Let’s consider for now only 1D separable interpolation and the range of 0.0-0.5 (-0.5 to 0.0 or 0.5 to 1.0 can be considered as switching to different sample pair, or symmetric). With a subpixel offset of 0.0, linear weights will be [1.0, 0.0], and with a subpixel offset of 0.5, they will be [0.5, 0.5]. Those two extremes correspond to a very different image filters! Imagine input filtered with a convolution filter of 1 (this returns just the original one) vs a filter of [0.5, 0.5] – this corresponds to low quality, box blur of the whole image. To demonstrate this effect, I made a shadertoy that compares bilinear, bicubic bspline, biquadratic, and a sinc filter (more about it later) with dynamically changing offsets. Same texture bilinearly interpolated. Upper left half of the image includes shift by 0.5 pixels, the other one no (and thus no filtering). I upsampled the resulting image with NN interpolation for magnification. So with an offset of U or V of 0.5, we are introducing very strong blurring (even more with both shifted – see above), and no blurring at all with no fractional offsets. Knowing that our effective image filters change based on the pixel “phase” we can have a look at spectral properties and frequency responses of those filters and see what we can learn about those resampling filters. Bilinear artifacts – variance loss, and frequency perspective So let’s have a look at filter weights of our 3 different types of filters as they change with pixel phase / fractional offset: As expected, they are symmetric, and we can verify that they sum to 1. Weights are quite interesting themselves – for example quadratic interpolation with fractional offset of 0.0 has the same filter weights as the linear interpolation with offset of 0.5! Looking at fractional offset dependent, we can analyze how those weights affect the signal variance, as well as the frequency response (and in effect, frequency content of the filtered images). Variance change resulting from a linear filter is equal to the sum of the squares of all its weights. So let’s plot the effect on input signal variance from those three filters. Position dependent variance reduction of three different analyzed filters. With the bilinear filter, the total variance changes (a lot!) with the subpixel position – which will cause apparent contrast loss. Variance and contrast loss due to blending/filtering is an interesting statistical effect, very visible perceptually, and one of my favorite papers on procedural texture synthesis from Eric Heitz and Fabrice Neyret achieves very good results from blending random blended tilings of an input texture mostly “just” by designing a variance preserving blending operator! Now, moving to the full frequency response of a those filters – for this we can use for example Z-transform commonly used in signal processing (or go directly with complex phasors). This is goes beyond scope of this blog post, (if you are interested in details, let me know in the comments!) but I will present the resulting frequency responses of those 3 different filters: Frequency responses of linear, quadratic, and bspline at fractional offsets of 0.0, 1/6, 1/3, 1/2. Each plot corresponds to a different fractional coordinate offset. Looking at those plots we can observe a very different behavior. As we realized above, linear at no fractional offset produces no filtering effect at all (identity response), and both other filters are lowpass filters. Interestingly, no offset is where quadratic interpolation performs the most lowpass filtering, the opposite of both other filters! To be able to reason about the filters in more of isolation, I also plotted each filters frequency response at different phases on a single plot for each: Frequency responses of linear, quadratic, and bspline at fractional offsets of 0.0, 1/6, 1/3, 1/2. Each plot corresponds to a different filter. Looking at these plots, we can see that bspline cubic is the most “consistent” between the different fractional offsets, while quadratic is less consistent, but filters out somewhat less of high frequencies (which is part of its sharper look). Linear is the least consistent, going from between a very sharp image, and a very blurry one. This inconsistency (both in terms of total variance, as well as frequency response – in fact the first can be derived from the latter from Parseval’s theorem) is another look at what we perceive as bilinear artifacts – “pulsing” and inconsistent filtering of the image. Looking at linear image shifted with phase 0.0 and 0.5 is like looking at two very different images. While bicubic bspline over-blurs the input, it is consistent, doesn’t “pulse” with changing offsets and doesn’t create aliased visible patterns. I see the biquadratic filter as kind of a “middle ground” that looks very reasonable and definitely will use it in practice. In the original eye screenshot texture we can see filtering / variance reduction in action. Pixels close to the interpolated texel origins are sharp (green), some get blurred horizontally (purple), some vertically (yellow), and some both (red). Such an non-uniform effect of variable blurring and variable contrast is yet another take on the bilinear artifacts. Given how bicubic and similar filters have quite consistent lowpass filtering, some of the blurring effect can be compensated with an additional, sharpening filter. This is what for example my friend Michal Drobot proposed in his talk on TAA for Far Cry 4 (and others have arrived to and used independently) – after resampling, do a global uniform sharpening pass on the whole image to counter some of the blurriness. Given how blurriness is relatively constant, sharpen filter can be even designed to recover the missing input frequencies accurately! Bilinear / bicubic alternatives – (windowed) sinc It’s worth noting that resampling filters don’t have to necessarily so heavily blur out the images and their frequency response can be much more high frequency preserving. This is beyond scope of this blog post, but some bicubic filters are actually designed to be slightly sharpening and boosting some of the high frequencies (Catmull-Rom spline – often used in TAA for continuous resampling). Such built-in sharpening might not be desired, so many filters are designed to be as frequency preserving as possible (windowed sinc like Lanczos). As I mentioned, it is beyond the scope of this blog post, but just for fun, I include example frequency response of a truncated (non windowed!) sinc, as well as final gif comparison. Such a sinc is also featured in my shadertoy. Unwidowed sinc frequency response at different subpixel offsets – note how there is no dominant lowpass filtering (with exception of last part, close to Nyquist), but we get lots of frequency response ringing and we will observe some oversharpening. Do not use unwindowed sinc in practice! All filters compared. Note that truncated sinc might be the sharpest, but it looks unacceptably (ugly!) with over-filtering and significantly stronger image contrast. If you’re looking for sharpness, better choice might be some tweaked bicubic filter, or windowed sinc – Lanczos. For much more complete and comprehensive treatment of filter trade-offs (sharpness, ringing, oversharpening), I again highly recommend this classic paper from Mitchell and Netravali – be sure to check it out. Conclusions This post touched on lots of topics and connection between them (I hope that it could be inspiring for you to go deeper into them) – I started with analyzing the most common, simple, ardware accelerated image interpolation filter – bilinear. I discussed its limitations and why it is often replaced (especially when interpolating from very low resolution textures) by other filters like bicubic – which might seem counter-intuitive given that they are much blurrier. I discussed what causes the “bilinear artifacts”, both from the simple spatial filtering perspective, as well as the effect of variance loss / contrast reduction / lowpass filtering. I barely scratched surface of the topic of interpolation, but I had fun writing it (and creating the visualizations), so expect some follow up posts about it in the future! Share this: Twitter Facebook Like this: Like Loading... This entry was posted in Code / Graphics and tagged blur, filtering, image processing, postprocessing, temporal, temporal supersampling. Bookmark the permalink. ← Using JAX, numpy, and optimization techniques to improve separable image filters Leave a Reply Cancel reply Enter your comment here... Please log in using one of these methods to post your comment: Email (required) (Address never made public) Name (required) Website You are commenting using your WordPress.com account. ( Log Out / Change ) You are commenting using your Google account. ( Log Out / Change ) You are commenting using your Twitter account. ( Log Out / Change ) You are commenting using your Facebook account. ( Log Out / Change ) Cancel Connecting to %s var highlander_expando_javascript = function(){ var input = document.createElement( 'input' ), comment = jQuery( '#comment' ); if ( 'placeholder' in input ) { comment.attr( 'placeholder', jQuery( '.comment-textarea label' ).remove().text() ); } // Expando Mode: start small, then auto-resize on first click + text length jQuery( '#comment-form-identity' ).hide(); jQuery( '#comment-form-subscribe' ).hide(); jQuery( '#commentform .form-submit' ).hide(); comment.css( { 'height':'10px' } ).one( 'focus', function() { var timer = setInterval( HighlanderComments.resizeCallback, 10 ) jQuery( this ).animate( { 'height': HighlanderComments.initialHeight } ).delay( 100 ).queue( function(n) { clearInterval( timer ); HighlanderComments.resizeCallback(); n(); } ); jQuery( '#comment-form-identity' ).slideDown(); jQuery( '#comment-form-subscribe' ).slideDown(); jQuery( '#commentform .form-submit' ).slideDown(); }); } jQuery(document).ready( highlander_expando_javascript ); Notify me of new comments via email. Notify me of new posts via email. Recent Posts Bilinear texture filtering – artifacts, alternatives, and frequency domain analysis April 14, 2020 Using JAX, numpy, and optimization techniques to improve separable image filters March 15, 2020 Separate your filters! Separability, SVD and low-rank approximation of 2D image processing filters February 3, 2020 Analyze your own activity data using Google Takeout – music listening stats example January 6, 2020 Local linear models and guided filtering – an alternative to bilateral filter September 22, 2019 Archives April 2020 (1) March 2020 (1) February 2020 (1) January 2020 (1) September 2019 (1) August 2019 (1) May 2018 (1) October 2017 (1) August 2017 (1) April 2017 (2) October 2016 (4) September 2016 (2) August 2016 (1) June 2016 (1) October 2015 (2) March 2015 (2) February 2015 (1) December 2014 (1) October 2014 (1) September 2014 (4) August 2014 (4) July 2014 (2) June 2014 (2) May 2014 (2) April 2014 (3) March 2014 (5) February 2014 (2) January 2014 (3) Categories Categories Select Category Code / Graphics Travel / Photography Uncategorized /* 0 ) { dropdown.parentNode.submit(); } } dropdown.onchange = onCatChange; })(); /* ]]> */ Recent Posts Bilinear texture filtering – artifacts, alternatives, and frequency domain analysis April 14, 2020 Using JAX, numpy, and optimization techniques to improve separable image filters March 15, 2020 Separate your filters! Separability, SVD and low-rank approximation of 2D image processing filters February 3, 2020 Analyze your own activity data using Google Takeout – music listening stats example January 6, 2020 Local linear models and guided filtering – an alternative to bilateral filter September 22, 2019 Follow me on TwitterMy Tweets Archives April 2020 March 2020 February 2020 January 2020 September 2019 August 2019 May 2018 October 2017 August 2017 April 2017 October 2016 September 2016 August 2016 June 2016 October 2015 March 2015 February 2015 December 2014 October 2014 September 2014 August 2014 July 2014 June 2014 May 2014 April 2014 March 2014 February 2014 January 2014 Bart Wronski Blog at WordPress.com. var WPGroHo = {"my_hash":""}; //initialize and attach hovercards to all gravatars jQuery( document ).ready( function( $ ) { if (typeof Gravatar === "undefined"){ return; } if ( typeof Gravatar.init !== "function" ) { return; } Gravatar.profile_cb = function( hash, id ) { WPGroHo.syncProfileData( hash, id ); }; Gravatar.my_hash = WPGroHo.my_hash; Gravatar.init( 'body', '#wp-admin-bar-my-account' ); }); var HighlanderComments = {"loggingInText":"Logging In\u2026","submittingText":"Posting Comment\u2026","postCommentText":"Post Comment","connectingToText":"Connecting to %s","commentingAsText":"%1$s: You are commenting using your %2$s account.","logoutText":"Log Out","loginText":"Log In","connectURL":"https:\/\/bartwronski.wordpress.com\/public.api\/connect\/?action=request&domain=bartwronski.com","logoutURL":"https:\/\/bartwronski.wordpress.com\/wp-login.php?action=logout&_wpnonce=11fc88cfc1","homeURL":"https:\/\/bartwronski.com\/","postID":"3736","gravDefault":"mystery","enterACommentError":"Please enter a comment","enterEmailError":"Please enter your email address here","invalidEmailError":"Invalid email address","enterAuthorError":"Please enter your name here","gravatarFromEmail":"This picture will show whenever you leave a comment. Click to customize it.","logInToExternalAccount":"Log in to use details from one of these accounts.","change":"Change","changeAccount":"Change Account","comment_registration":"1","userIsLoggedIn":"","isJetpack":"","text_direction":"ltr"}; Post to Cancel window.WPCOM_sharing_counts = {"https:\/\/bartwronski.com\/2020\/04\/14\/bilinear-texture-filtering-artifacts-alternatives-and-frequency-domain-analysis\/":3736}; var actionbardata = {"siteID":"62253954","siteName":"Bart Wronski","siteURL":"https:\/\/bartwronski.com","icon":"","canManageOptions":"","canCustomizeSite":"","isFollowing":"","themeSlug":"pub\/twentyten","signupURL":"https:\/\/wordpress.com\/start\/","loginURL":"https:\/\/wordpress.com\/log-in?redirect_to=https%3A%2F%2Fbartwronski.com%2F2020%2F04%2F14%2Fbilinear-texture-filtering-artifacts-alternatives-and-frequency-domain-analysis%2F&signup_flow=account&domain=bartwronski.com","themeURL":"","xhrURL":"https:\/\/bartwronski.com\/wp-admin\/admin-ajax.php","nonce":"d12b6445f0","isSingular":"1","isFolded":"","isLoggedIn":"","isMobile":"","subscribeNonce":"","referer":"https:\/\/bartwronski.com\/2020\/04\/14\/bilinear-texture-filtering-artifacts-alternatives-and-frequency-domain-analysis\/","canFollow":"1","feedID":"16883079","statusMessage":"","customizeLink":"https:\/\/bartwronski.wordpress.com\/wp-admin\/customize.php?url=https%3A%2F%2Fbartwronski.wordpress.com%2F2020%2F04%2F14%2Fbilinear-texture-filtering-artifacts-alternatives-and-frequency-domain-analysis%2F","postID":"3736","shortlink":"https:\/\/wp.me\/p4dd62-Yg","canEditPost":"","editLink":"https:\/\/wordpress.com\/post\/bartwronski.com\/3736","statsLink":"https:\/\/wordpress.com\/stats\/post\/3736\/bartwronski.com","i18n":{"view":"View site","follow":"Follow","following":"Following","edit":"Edit","login":"Log in","signup":"Sign up","customize":"Customize","report":"Report this content","themeInfo":"Get theme: Twenty Ten","shortlink":"Copy shortlink","copied":"Copied","followedText":"New posts from this site will now appear in your Reader ","foldBar":"Collapse this bar","unfoldBar":"Expand this bar","editSubs":"Manage subscriptions","viewReader":"View site in Reader","viewReadPost":"View post in Reader","subscribe":"Sign me up","enterEmail":"Enter your email address","followers":"Join 347 other followers","alreadyUser":"Already have a WordPress.com account? Log in now. ","stats":"Stats"}}; var jetpackCarouselStrings = {"widths":[370,700,1000,1200,1400,2000],"is_logged_in":"","lang":"en","ajaxurl":"https:\/\/bartwronski.com\/wp-admin\/admin-ajax.php","nonce":"2cb7a29065","display_exif":"1","display_geo":"1","single_image_gallery":"1","single_image_gallery_media_file":"","background_color":"black","comment":"Comment","post_comment":"Post Comment","write_comment":"Write a Comment...","loading_comments":"Loading Comments...","download_original":"View full size {0}\u00d7 {1} ","no_comment_text":"Please be sure to submit some text with your comment.","no_comment_email":"Please provide an email address to comment.","no_comment_author":"Please provide your name to comment.","comment_post_error":"Sorry, but there was an error posting your comment. Please try again later.","comment_approved":"Your comment was approved.","comment_unapproved":"Your comment is in moderation.","camera":"Camera","aperture":"Aperture","shutter_speed":"Shutter Speed","focal_length":"Focal Length","copyright":"Copyright","comment_registration":"1","require_name_email":"1","login_url":"https:\/\/bartwronski.wordpress.com\/wp-login.php?redirect_to=https%3A%2F%2Fbartwronski.com%2F2020%2F04%2F14%2Fbilinear-texture-filtering-artifacts-alternatives-and-frequency-domain-analysis%2F","blog_id":"62253954","meta_data":["camera","aperture","shutter_speed","focal_length","copyright"],"local_comments_commenting_as":" You must be logged in to post a comment. ","reblog":"Reblog","reblogged":"Reblogged","reblog_add_thoughts":"Add your thoughts here... (optional)","reblogging":"Reblogging...","post_reblog":"Post Reblog","stats_query_args":"blog=62253954&v=wpcom&tz=-7&user_id=0&subd=bartwronski","is_public":"1","reblog_enabled":""}; var sharing_js_options = {"lang":"en","counts":"1","is_stats_active":"1"}; ( 'fetch' in window ) || document.write( '' );( document.contains ) || document.write( '' );( window.DOMRect ) || document.write( '' );( window.URL && window.URL.prototype && window.URLSearchParams ) || document.write( '' );( window.FormData && window.FormData.prototype.keys ) || document.write( '' );( Element.prototype.matches && Element.prototype.closest ) || document.write( '' ); ( window.URL && window.URL.prototype && window.URLSearchParams ) || document.write( '' ); ( window.DOMRect ) || document.write( '' ); var windowOpen; jQuery( document.body ).on( 'click', 'a.share-twitter', function() { // If there's another sharing window open, close it. if ( 'undefined' !== typeof windowOpen ) { windowOpen.close(); } windowOpen = window.open( jQuery( this ).attr( 'href' ), 'wpcomtwitter', 'menubar=1,resizable=1,width=600,height=350' ); return false; }); var windowOpen; jQuery( document.body ).on( 'click', 'a.share-facebook', function() { // If there's another sharing window open, close it. if ( 'undefined' !== typeof windowOpen ) { windowOpen.close(); } windowOpen = window.open( jQuery( this ).attr( 'href' ), 'wpcomfacebook', 'menubar=1,resizable=1,width=600,height=400' ); return false; }); // %d bloggers like this: _tkq = window._tkq || []; _stq = window._stq || []; _tkq.push(['storeContext', {'blog_id':'62253954','blog_tz':'-7','user_lang':'en','blog_lang':'en','user_id':'0'}]); _stq.push(['view', {'blog':'62253954','v':'wpcom','tz':'-7','user_id':'0','post':'3736','subd':'bartwronski'}]); _stq.push(['extra', {'crypt':'UE40eW5QN0p8M2Y/RE1TaVhzUzFMbjdWNHpwZGhTayxPSUFCMGRVYVNrSFguN3FwSmQ5RGtNX3VQcj1yVzhiflM1THQtLGFdQ2toOXYlQjYrMC9LMGRJJjR6ajd1Lm9nQXNlX3RUODVsUjZzM1BLQVs1aC4zWTR5Z3g2LjEmeGxFQyZ0RkZfM0VjLERZWUpJa1YlTThHaFJHTis2ek9YVVM5XUdUXzRTSS8wS3RGTlZXeVRKai4xXTA0M284RUlrTHUmfFBkbW9JL0FYaE5KTzFXaywuVVtrP1p+dWpqUFszL3BwSVI2dzZwRGp6PWRbOWFlJmltZ2hUeXN2SGNSSHBndEpxWj9zam9bNnRKOFl2RHc/NUo1N05KY3BjWTd2TSxUd3V3V0NjQ1JDcVd5SnxiTSYsdC15cCZ4eVUmQQ=='}]); _stq.push([ 'clickTrackerInit', '62253954', '3736' ]); if ( 'object' === typeof wpcom_mobile_user_agent_info ) { wpcom_mobile_user_agent_info.init(); var mobileStatsQueryString = ""; if( false !== wpcom_mobile_user_agent_info.matchedPlatformName ) mobileStatsQueryString += "&x_" + 'mobile_platforms' + '=' + wpcom_mobile_user_agent_info.matchedPlatformName; if( false !== wpcom_mobile_user_agent_info.matchedUserAgentName ) mobileStatsQueryString += "&x_" + 'mobile_devices' + '=' + wpcom_mobile_user_agent_info.matchedUserAgentName; if( wpcom_mobile_user_agent_info.isIPad() ) mobileStatsQueryString += "&x_" + 'ipad_views' + '=' + 'views'; if( "" != mobileStatsQueryString ) { new Image().src = document.location.protocol + '//pixel.wp.com/g.gif?v=wpcom-no-pv' + mobileStatsQueryString + '&baba=' + Math.random(); } }

Bilinear Texture Filtering – Artifacts, Alternatives, and Frequency Domain Analysis

Comments

Recommended Tutorials

Other Tutorials by GameDev.net

Bilinear Texture Filtering – Artifacts, Alternatives, and Frequency Domain Analysis

Comments

Recommended Tutorials

Other Tutorials by GameDev.net

Reticulating splines