Normally SharePoint 2010 Lookups open in a new tab. So if you want to open it as a popup, it is really simple. Add fallowing script to your SharePoint page.
<script type="text/javascript">
var winIsDlg = 0;
(function () {
var e, q = window.location.search.substring(1), r = /([^&=]+)=([^&]+)/g;
while (e = r.exec(q))
if(decodeURIComponent(e[1])=="IsDlg" && decodeURIComponent(e[2]) == "1")
{winIsDlg=1;}
})();
$(document).ready(function(){
if(winIsDlg==1)
{
$("td[id='SPFieldLookup'] a").each(
function(){
this.href="javascript:OpenPopUpPage('" + this.href + "')";
});
}
});
</script>