Comments Page (different when no comments)

I’m having issues with comments page of the theme I’m working on. It looks fine when there are comments, but the footer moves to the top when there aren’t any.

With Comments.
Without Comments.

This is the code for the comments page. If you need any more information, just let me know.

<?php

	if (!empty($_SERVER['SCRIPT_FILENAME']) && 'comments.php' == basename($_SERVER['SCRIPT_FILENAME']))
		die ('Please do not load this page directly. Thanks!');

	if ( post_password_required() ) { ?>
		<p class="nocomments">This post is password protected. Enter the password to view comments.</p>
	<?php
		return;
	}
?>

<div class="bob">
<?php if ( have_comments() ) : ?>
	<h3><?php comments_number('No Responses', 'One Response', '% Responses' );?> to “<?php the_title(); ?>”</h3>

	<div class="bobbox">

		<div class="navigation">
		<div class="alignleft"><?php previous_comments_link() ?></div>
		<div class="alignright"><?php next_comments_link() ?></div>
	</div>

	<ol class="ugh">
	<?php wp_list_comments(); ?>
	</ol>

	<div class="navigation">
		<div class="alignleft"><?php previous_comments_link() ?></div>
		<div class="alignright"><?php next_comments_link() ?></div>
	</div>

	<?php else : // this is displayed if there are no comments so far ?>

	<?php if ( comments_open() ) : ?>

	 <?php else : // comments are closed ?>
		<p class="nocomments">Comments are closed.</p>

<?php endif; ?>
<?php endif; ?>

<?php if ( comments_open() ) : ?>

<h3 class="other"><?php comment_form_title( 'Leave a Reply', 'Leave a Reply to %s' ); ?></h3><br />

<div class="cancel-comment-reply">
	<small><?php cancel_comment_reply_link(); ?></small>
</div>

<?php if ( get_option('comment_registration') && !is_user_logged_in() ) : ?>
<p>You must be <a href="<?php echo wp_login_url( get_permalink() ); ?>">logged in</a> to post a comment.</p>
<?php else : ?>

<form action="<?php echo get_option('siteurl'); ?>/wp-comments-post.php" method="post" id="commentform">

<?php if ( is_user_logged_in() ) : ?>

<p>Logged in as <a href="<?php echo get_option('siteurl'); ?>/wp-admin/profile.php"><?php echo $user_identity; ?></a>. <a href="<?php echo wp_logout_url(get_permalink()); ?>" title="Log out of this account">Log out &raquo;</a></p>

<?php else : ?>

<p><input class="commentform" type="text" name="author" id="author" value="<?php echo ($comment_author); ?>" size="22" tabindex="1" <?php if ($req) echo "aria-required='true'"; ?> />
<label for="author"><small>Name <?php if ($req) echo ""; ?></small></label></p>

<p><input class="commentform" type="text" name="email" id="email" value="<?php echo $comment_author_email; ?>" size="22" tabindex="2" <?php if ($req) echo "aria-required='true'"; ?> />
<label for="email"><small>Email<?php if ($req) echo ""; ?></small></label></p>

<p><input class="commentform" type="text" name="url" id="url" value="<?php echo $comment_author_url; ?>" size="22" tabindex="3" />
<label for="url"><small>Website</small></label></p>

<?php endif; ?>

<p><textarea class="commentform" name="comment" id="comment" cols="100%" rows="10" tabindex="4"></textarea></p>

<p><center><input class="commentform2" name="submit" type="submit" id="submit" tabindex="5" value="Submit Comment" /></center>
<?php comment_id_fields(); ?>
</p>

</form>

<?php endif;?>

</div>

<?php endif;?>

</div>

<?php get_footer(); ?>