Modal popup using xslt
<a class="clsTitle" onclick="javascript:SP.UI.ModalDialog.ShowPopupDialog('{$SafeLinkUrl}'); return false;" title="{@LinkToolTip}">
<xsl:if test="$ItemsHaveStreams = 'True'">
<!--code for modal popup -->
<xsl:template name="ModalDialogPopUp" match="Row[@Style='ModalDialogPopUp']" mode="itemstyle">
<xsl:variable name="SafeLinkUrl">
<xsl:call-template name="OuterTemplate.GetSafeLink">
<xsl:with-param name="UrlColumnName" select="'LinkUrl'"/>
</xsl:call-template>
</xsl:variable>
<xsl:variable name="DisplayTitle">
<xsl:call-template name="OuterTemplate.GetTitle">
<xsl:with-param name="Title" select="@Title"/>
<xsl:with-param name="UrlColumnName" select="'LinkUrl'"/>
</xsl:call-template>
</xsl:variable>
<div class="item link-item bullet">
<xsl:call-template name="OuterTemplate.CallPresenceStatusIconTemplate"/>
<a onclick="javascript:SP.UI.ModalDialog.ShowPopupDialog('{$SafeLinkUrl}'); return false;" onmouseover="javascript:this.style.cursor='hand';" title="{@LinkToolTip}">
<xsl:value-of select="$DisplayTitle"/>
</a>
</div>
</xsl:template>
<!-- End of modal popup code -->